commit a0f3248e4a1224f209340886372f0cb8d67eed26 Author: ModelHub XC Date: Sat Aug 22 12:19:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: lxazjk/qwen2.5-1.5b-24game-grpo Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..de17285 --- /dev/null +++ b/README.md @@ -0,0 +1,132 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen2.5-1.5B-Instruct +library_name: transformers +pipeline_tag: text-generation +language: +- en +tags: +- grpo +- rlvr +- trl +- reasoning +- 24-game +- countdown +- verifier-guided-sampling +--- + +# Qwen2.5-1.5B-Instruct · GRPO/RLVR for the 24 Game + +GRPO-trained `Qwen/Qwen2.5-1.5B-Instruct` that solves the **24 game** (and, by +transfer, **Countdown**) in R1 style. Rewards come from a **deterministic program +verifier** (RLVR) — no neural reward model. This is the `shaped`-reward GRPO +checkpoint (300 steps, from an SFT warmup). + +Output format: + +```text +reasoning3*8 +``` + +The `` may only contain the input numbers, `+ - * /`, and parentheses; +each input number is used exactly once; the value must equal the target (24 by +default, arbitrary for Countdown). + +## TL;DR — how to use it + +> **This model is a verifier-guided *sampler*, not a reliable one-shot solver.** +> Greedy/pass@1 ≈ 0. Its value is that **sampling many candidates and letting the +> verifier pick a correct one** works very well. Use temperature ~1.1, sample N, +> then run the program verifier to select the first valid expression. + +## Results (100 hardest problems, human-lowest-solved-rate split) + +| Metric | This model | Note | +| --- | ---: | --- | +| greedy / pass@1 | **0/100** | GRPO does not improve one-shot accuracy | +| sample@16 (per independent run) | **~36/100** (max 43) | temp 1.1, top-p 0.95 | +| best-of-128 (single model, verifier-selected) | **74/100** | 128 candidates/problem | +| union best-of-runs (this + correctness-heavy, 256/problem) | **79/100** | two GRPO checkpoints merged | +| **adaptive best-of-runs** | **100/100** | escalate sampling only on unsolved; 60,416 total candidates, hardest problem needed 5,376 | + +Diagnostics on first attempts: `r1_format = 1.00`, `numbers_ok = 0.99` — the model +reliably learned the format and exact number-usage constraint, but not exact +arithmetic search. That is why test-time sampling + verifier selection is the +right way to use it. + +### Countdown (bonus, arbitrary target) + +| Setting | greedy | sample@16 | +| --- | ---: | ---: | +| this 24-game checkpoint, zero-shot | 1/100 | 11/100 | +| after 50-step target-aware Countdown GRPO | 1/100 | 18/100 | + +## Usage + +```python +import re +from fractions import Fraction +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +name = "lxazjk/qwen2.5-1.5b-24game-grpo" +tok = AutoTokenizer.from_pretrained(name) +model = AutoModelForCausalLM.from_pretrained(name, torch_dtype="auto", device_map="auto") + +numbers = [2, 4, 7, 7] +prompt = ( + "You solve arithmetic target games. Use each given number exactly once. " + "Only use +, -, *, / and parentheses. Think briefly, then put the final " + "expression inside .... The answer must be only an " + "expression, with no equals sign, result value, example, or explanation.\n\n" + f"Make 24.\nNumbers: {', '.join(map(str, numbers))}\n" + "Respond in this exact format:\nreasoning stepsexpression only" +) + +# sample many candidates, then verify +ids = tok(prompt, return_tensors="pt").to(model.device) +out = model.generate(**ids, do_sample=True, temperature=1.1, top_p=0.95, + num_return_sequences=16, max_new_tokens=128) +cands = tok.batch_decode(out[:, ids.input_ids.shape[1]:], skip_special_tokens=True) + +def verify(expr, nums, target=24): + toks = re.findall(r"\d+", expr) + if sorted(map(int, toks)) != sorted(nums): + return False + if re.fullmatch(r"[0-9+\-*/() ]+", expr) is None: + return False + try: + # evaluate with exact rational arithmetic + val = eval(expr, {"__builtins__": {}}) + return Fraction(val) == target + except Exception: + return False + +for c in cands: + m = re.search(r"(.*?)", c, re.S) + if m and verify(m.group(1).strip(), numbers): + print("solved:", m.group(1).strip()); break +``` + +(In the original project the verifier uses a safe AST whitelist + `Fraction`; the +snippet above is a compact stand-in. Never `eval` untrusted strings in production.) + +## Training + +- Base: `Qwen/Qwen2.5-1.5B-Instruct` +- SFT warmup (format + number usage), then **GRPO** with a rule-based reward set + (`format / structure / expression / number-usage / closeness / correctness`). +- Verifier: extract last ``, restrict characters, check number multiset, + parse via a safe AST whitelist, evaluate with `Fraction`, compare to target. +- Data: `nlile/24-game` (train), `test-time-compute/game-of-24` (hard eval), + `Jiayi-Pan/Countdown-Tasks-3to4` (bonus). + +See `results/` in this repo for the full per-setting evaluation summaries +(`REPRODUCTION_SUMMARY.md`, greedy / sample / union / adaptive / Countdown JSONs). + +## Limitations + +- Not a one-shot solver — high solve rates require substantial test-time sampling. +- The hard-eval number combinations overlap the training set, so this is a hard + eval, not a strict OOD generalization test. +- `100/100` is a test-time sampling + verifier-selection result, **not** pass@1. diff --git a/config.json b/config.json new file mode 100644 index 0000000..d9648b4 --- /dev/null +++ b/config.json @@ -0,0 +1,61 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.12.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..8ae691b --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.12.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..82a8f5e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1701598bc877732c2d9f7234cc82029ae5ec075894b778468d586782b6eabc9f +size 3087467144 diff --git a/results/REPRODUCTION_SUMMARY.md b/results/REPRODUCTION_SUMMARY.md new file mode 100644 index 0000000..58f2f84 --- /dev/null +++ b/results/REPRODUCTION_SUMMARY.md @@ -0,0 +1,44 @@ +# 24-game RL 复现结果汇总（从零完整跑通） + +环境: 8×H20 (97GB). transformers 5.12.1 / trl 1.4.0 / datasets 5.0.0. 代理走 sys-proxy. +本文件所有产物在 outputs/（gitignored），不污染被并发修改的仓库。 + +## 流水线（全部完成） +1. 数据: data.build(24) + data.sft + data.build(countdown) → train 1362 / sft 1262 / eval 100 / countdown 1000+100 +2. Oracle sanity: 24-game 100/100, countdown 100/100 +3. SFT warmup: configs/sft.yaml, 790 步, ~4.7 min, eval_loss 0.119 +4. GRPO ×2（并行）: shaped 300步(~7min, reward 0.92→1.07) / correctness_heavy 200步(~5min, reward 0.85→0.91)，均从 SFT checkpoint +5. 评测: greedy + 8卡分片 sample@16 → best-of-128 → 跨模型 union → adaptive +6. Countdown bonus: 从 ch checkpoint 续训 50 步 + zero-shot / trained 评测 +7. 复验: 20 单测通过, compileall 通过 + +## 24-game 主任务结果 vs report + +| 指标 | 本次复现 | report | 一致性 | +|---|---|---|---| +| greedy / pass@1 (base/sft/shaped/ch) | 0 / 0 / 0 / 0 /100 | ~0–3/100 | ✅ GRPO 不提升 greedy | +| sample@16 (shaped, 8 shard 均值/最大) | ~36 / 43 | 38 | ✅ | +| sample@16 (ch, 均值/最大) | ~34 / 40 | 45 | ≈（report 用 step50 甜点） | +| best-of-128 单模型 (shaped / ch) | 74 / 71 | ~78 (sample@128) | ✅ | +| union best-of-runs (2模型, 256候选/题) | 79 | 89 (base best-of) | ✅ 同区间 | +| adaptive best-of-runs | **100/100** | 100/100 | ✅ | +| adaptive 总候选 / 最难题预算 | 60,416 / 5,376 | 76,240 / 8,212 | ✅ 同量级 | + +r1_format=1.00, numbers_ok=0.99（模型学会了格式与数字使用，但没学会精确算式搜索）→ 与 report 结论一致：**verifier-guided sampler，而非稳定 1-shot solver。** + +## Countdown bonus 结果 vs report + +| 设置 | greedy | sample@16 | report sample@16 | +|---|---|---|---| +| Oracle | 100/100 | - | 100 | +| 24-game ckpt zero-shot | 1/100 | 11/100 | 12 | +| Countdown GRPO 50步 | 1/100 | 18/100 | 16 | + +结论一致：24-game→任意 target 有弱迁移；短 GRPO 提升 sample@16 但不改 greedy。 + +## 关键产物路径 +- SFT: outputs/sft-qwen-24-strong/checkpoint-final +- GRPO: outputs/grpo-qwen-24-shaped / outputs/grpo-qwen-24-correctness-heavy /checkpoint-final +- Countdown: outputs/grpo-qwen-countdown-from-24/checkpoint-final +- 评测汇总 JSON: outputs/eval/*_summary.json, outputs/eval/adaptive/adaptive_final_summary.json +- 复用脚本: outputs/run_sample_eval.sh, outputs/adaptive_eval.sh, outputs/run_configs/*.yaml diff --git a/results/adaptive_final_summary.json b/results/adaptive_final_summary.json new file mode 100644 index 0000000..c223b29 --- /dev/null +++ b/results/adaptive_final_summary.json @@ -0,0 +1,1105 @@ +{ + "count": 100, + "solved": 100, + "solved_rate": 1.0, + "total_model_candidates": 60416, + "mean_candidate_budget_per_problem": 604.16, + "max_candidate_budget_per_problem": 5376, + "min_candidate_budget_per_problem": 256, + "mean_candidates_until_solution": 282.45, + "median_candidates_until_solution": 30.5, + "max_candidates_until_solution": 4774, + "source_budgets": { + "outputs/eval/sample_shaped/shard_0.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_1.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_2.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_3.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_4.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_5.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_6.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_7.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_0.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_1.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_2.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_3.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_4.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_5.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_6.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_7.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/adaptive/r1_shard_0.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_1.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_2.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_3.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_4.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_5.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_6.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r1_shard_7.jsonl": { + "rows": 21, + "candidates": 2688 + }, + "outputs/eval/adaptive/r2_shard_0.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_1.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_2.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_3.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_4.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_5.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_6.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r2_shard_7.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_0.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_1.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_2.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_3.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_4.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_5.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_6.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r3_shard_7.jsonl": { + "rows": 4, + "candidates": 512 + }, + "outputs/eval/adaptive/r4_shard_0.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_1.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_2.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_3.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_4.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_5.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_6.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r4_shard_7.jsonl": { + "rows": 3, + "candidates": 384 + }, + "outputs/eval/adaptive/r5_shard_0.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_1.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_2.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_3.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_4.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_5.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_6.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r5_shard_7.jsonl": { + "rows": 2, + "candidates": 256 + }, + "outputs/eval/adaptive/r6_shard_0.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_1.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_2.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_3.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_4.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_5.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_6.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r6_shard_7.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_0.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_1.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_2.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_3.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_4.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_5.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_6.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r7_shard_7.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_0.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_1.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_2.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_3.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_4.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_5.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_6.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r8_shard_7.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_0.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_1.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_2.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_3.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_4.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_5.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_6.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r9_shard_7.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_0.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_1.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_2.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_3.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_4.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_5.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_6.jsonl": { + "rows": 0, + "candidates": 0 + }, + "outputs/eval/adaptive/r10_shard_7.jsonl": { + "rows": 0, + "candidates": 0 + } + }, + "unsolved_keys": [], + "reason_counts": { + "value is 18, expected 24": 2717, + "value is 48, expected 24": 2423, + "value is 36, expected 24": 2122, + "value is 16, expected 24": 2084, + "value is 20, expected 24": 2016, + "value is 60, expected 24": 1817, + "value is 12, expected 24": 1694, + "value is 22, expected 24": 1690, + "value is 96, expected 24": 1634, + "value is 30, expected 24": 1595, + "value is 66, expected 24": 1577, + "value is 21, expected 24": 1387, + "value is 27, expected 24": 1341, + "empty expression": 1302, + "value is 40, expected 24": 1290, + "numbers are not used exactly once": 1277, + "value is 72, expected 24": 1179, + "value is 84, expected 24": 1155, + "value is 0, expected 24": 1088, + "ok": 1065, + "value is 8, expected 24": 993, + "value is 42, expected 24": 986, + "value is 64, expected 24": 966, + "value is 6, expected 24": 958, + "value is 25, expected 24": 949, + "value is 28, expected 24": 843, + "value is 26, expected 24": 790, + "value is 29, expected 24": 784, + "value is 14, expected 24": 769, + "value is 32, expected 24": 756, + "value is 10, expected 24": 649, + "value is 120, expected 24": 624, + "value is 19, expected 24": 555, + "value is 15, expected 24": 540, + "value is 23, expected 24": 528, + "value is 80, expected 24": 496, + "value is 33, expected 24": 461, + "value is 54, expected 24": 427, + "value is 70, expected 24": 426, + "value is 35, expected 24": 384, + "value is 45, expected 24": 373, + "value is 104, expected 24": 361, + "value is 39, expected 24": 345, + "value is 56, expected 24": 332, + "value is 13, expected 24": 310, + "value is 31, expected 24": 308, + "value is 52, expected 24": 303, + "value is -6, expected 24": 298, + "value is 25.6666666667, expected 24": 298, + "contains disallowed characters": 292, + "value is -26, expected 24": 280, + "value is 9, expected 24": 277, + "value is 34, expected 24": 263, + "value is -16, expected 24": 262, + "value is -12, expected 24": 243, + "value is 37, expected 24": 240, + "value is 43, expected 24": 223, + "value is 50, expected 24": 222, + "value is 4, expected 24": 214, + "value is 44, expected 24": 186, + "value is 38, expected 24": 186, + "value is -22, expected 24": 175, + "value is 17, expected 24": 166, + "value is 97, expected 24": 159, + "value is 5.71428571429, expected 24": 148, + "value is 108, expected 24": 135, + "value is 21.5, expected 24": 128, + "value is -7, expected 24": 127, + "value is 9.33333333333, expected 24": 122, + "value is 24.6153846154, expected 24": 121, + "value is 90, expected 24": 120, + "value is 16.5, expected 24": 120, + "division by zero": 120, + "value is 7, expected 24": 118, + "value is 98, expected 24": 118, + "value is 78, expected 24": 115, + "value is 63, expected 24": 115, + "value is 11, expected 24": 114, + "value is 82.5, expected 24": 114, + "value is 10.5, expected 24": 104, + "value is 5, expected 24": 101, + "value is 13.3333333333, expected 24": 101, + "value is 7.61538461538, expected 24": 85, + "value is 18.6666666667, expected 24": 84, + "value is 77, expected 24": 83, + "value is 22.5, expected 24": 81, + "value is 16.6666666667, expected 24": 78, + "value is 6.54545454545, expected 24": 74, + "value is 65, expected 24": 72, + "value is 51.0769230769, expected 24": 70, + "value is 3, expected 24": 68, + "value is 6.4, expected 24": 66, + "value is 10.8, expected 24": 62, + "value is 25.5, expected 24": 60, + "value is 7.25, expected 24": 59, + "value is 75, expected 24": 57, + "value is 117, expected 24": 56, + "value is 6.66666666667, expected 24": 55, + "value is 20.5714285714, expected 24": 49, + "value is 12.8571428571, expected 24": 49, + "value is -8, expected 24": 48, + "value is 24.75, expected 24": 48, + "value is 21.3333333333, expected 24": 47, + "value is 33.3333333333, expected 24": 46, + "value is 91, expected 24": 45, + "value is 49, expected 24": 45, + "value is 110, expected 24": 44, + "value is -65, expected 24": 43, + "value is 160, expected 24": 43, + "value is 10.6666666667, expected 24": 42, + "value is 23.2727272727, expected 24": 42, + "value is 12.5, expected 24": 41, + "value is 21.6666666667, expected 24": 41, + "value is -20, expected 24": 40, + "value is -36, expected 24": 39, + "value is 23.375, expected 24": 39, + "value is 88, expected 24": 38, + "value is 47, expected 24": 37, + "value is 9.6, expected 24": 37, + "value is 14.7777777778, expected 24": 37, + "value is 31.5, expected 24": 34, + "value is 23.1111111111, expected 24": 34, + "value is 5.6, expected 24": 34, + "value is 144, expected 24": 33, + "value is 126, expected 24": 33, + "value is 26.4, expected 24": 33, + "value is 99, expected 24": 32, + "value is 186, expected 24": 32, + "value is 78.2222222222, expected 24": 32, + "value is 13.5, expected 24": 31, + "value is 85.3333333333, expected 24": 31, + "value is 67, expected 24": 30, + "value is 79, expected 24": 30, + "value is 13.7142857143, expected 24": 29, + "value is 138, expected 24": 29, + "value is 8.27272727273, expected 24": 29, + "value is 7.63636363636, expected 24": 29, + "value is 46, expected 24": 28, + "value is 168, expected 24": 28, + "value is 53, expected 24": 27, + "value is 4.66666666667, expected 24": 27, + "value is 25.6, expected 24": 27, + "value is 13.6, expected 24": 27, + "value is 198, expected 24": 26, + "value is 24.3333333333, expected 24": 26, + "value is -15, expected 24": 25, + "value is 55, expected 24": 25, + "value is 182, expected 24": 24, + "value is 44.5714285714, expected 24": 24, + "value is -136, expected 24": 24, + "value is 7.09090909091, expected 24": 24, + "value is 86, expected 24": 23, + "value is 17.6, expected 24": 23, + "value is 24.2, expected 24": 23, + "value is 6.5, expected 24": 23, + "value is 32.5714285714, expected 24": 23, + "value is 81, expected 24": 23, + "value is 9.5, expected 24": 22, + "value is 14.4, expected 24": 22, + "value is 17.3333333333, expected 24": 21, + "value is 44.625, expected 24": 21, + "value is 26.6666666667, expected 24": 21, + "value is 52.25, expected 24": 21, + "numbers must be positive integers": 20, + "value is 140, expected 24": 20, + "value is 9.23076923077, expected 24": 20, + "value is 57, expected 24": 20, + "value is 41, expected 24": 19, + "value is 29.3333333333, expected 24": 19, + "value is 94.9090909091, expected 24": 19, + "value is 28.5, expected 24": 18, + "value is 22.4, expected 24": 18, + "value is -30, expected 24": 18, + "value is 25.8461538462, expected 24": 18, + "value is 4.5, expected 24": 17, + "value is 165, expected 24": 17, + "value is 4.61538461538, expected 24": 17, + "value is 70.5, expected 24": 17, + "value is 19.5555555556, expected 24": 17, + "value is 7.71428571429, expected 24": 16, + "value is 58.5, expected 24": 16, + "value is 24.5, expected 24": 16, + "value is 71, expected 24": 16, + "value is 11.2, expected 24": 16, + "value is 1.5, expected 24": 15, + "value is 210, expected 24": 15, + "value is 19.5, expected 24": 15, + "value is 27.1428571429, expected 24": 15, + "value is 22.8571428571, expected 24": 15, + "value is 61, expected 24": 15, + "value is 8.55555555556, expected 24": 15, + "value is -2, expected 24": 15, + "value is 17.1428571429, expected 24": 15, + "value is 240, expected 24": 14, + "value is -18, expected 24": 14, + "value is 28.8, expected 24": 14, + "value is 25.6923076923, expected 24": 14, + "value is 5.90909090909, expected 24": 14, + "value is 11.25, expected 24": 14, + "value is 53.1666666667, expected 24": 14, + "unsupported expression node: UnaryOp": 13, + "value is 10.4, expected 24": 13, + "value is 45.5, expected 24": 13, + "value is 228, expected 24": 13, + "value is 5.14285714286, expected 24": 13, + "value is 21.6363636364, expected 24": 13, + "value is 15.4285714286, expected 24": 13, + "value is 93, expected 24": 13, + "value is 23.125, expected 24": 13, + "value is 13.2, expected 24": 12, + "value is 5.77777777778, expected 24": 12, + "value is 68, expected 24": 12, + "value is 65.5, expected 24": 12, + "value is 50.6666666667, expected 24": 12, + "value is 13.75, expected 24": 12, + "value is 30.6666666667, expected 24": 12, + "value is 114, expected 24": 11, + "value is 1.44444444444, expected 24": 11, + "value is 7.14285714286, expected 24": 11, + "value is 132, expected 24": 11, + "value is -24, expected 24": 11, + "value is 12.6666666667, expected 24": 11, + "value is 55.25, expected 24": 11, + "value is 20.0909090909, expected 24": 11, + "value is 56.5714285714, expected 24": 11, + "value is 69.3333333333, expected 24": 11, + "value is 9.71428571429, expected 24": 11, + "value is 14.5714285714, expected 24": 11, + "value is -4.45454545455, expected 24": 11, + "value is 22.9090909091, expected 24": 11, + "value is 8.4, expected 24": 10, + "value is -3, expected 24": 10, + "invalid syntax (, line 1)": 10, + "value is 20.5, expected 24": 10, + "value is -9, expected 24": 10, + "value is 112, expected 24": 10, + "value is 22.1428571429, expected 24": 10, + "value is 121, expected 24": 10, + "value is 18.3333333333, expected 24": 10, + "value is 23.5714285714, expected 24": 10, + "value is 8.33333333333, expected 24": 10, + "value is 17.5454545455, expected 24": 10, + "value is 1164, expected 24": 10, + "value is 105, expected 24": 9, + "value is 37.3333333333, expected 24": 9, + "value is 4.36363636364, expected 24": 9, + "value is 20.4285714286, expected 24": 9, + "value is 8.125, expected 24": 9, + "value is 150, expected 24": 9, + "value is 40.5, expected 24": 9, + "value is 17.8, expected 24": 9, + "value is 5.4, expected 24": 9, + "value is 23.2222222222, expected 24": 9, + "value is 62, expected 24": 8, + "value is 19.3333333333, expected 24": 8, + "value is -14, expected 24": 8, + "value is 14.8571428571, expected 24": 8, + "value is 2, expected 24": 8, + "value is 22.2857142857, expected 24": 8, + "value is 38.5, expected 24": 8, + "value is 6.85714285714, expected 24": 8, + "value is 136, expected 24": 8, + "value is 26.5, expected 24": 8, + "value is 2.66666666667, expected 24": 8, + "value is 8.66666666667, expected 24": 8, + "value is 22.6666666667, expected 24": 8, + "value is 8.25, expected 24": 8, + "value is 19.7142857143, expected 24": 8, + "value is 31.6666666667, expected 24": 8, + "value is 35.6, expected 24": 8, + "value is 28.3636363636, expected 24": 8, + "value is 29.1666666667, expected 24": 8, + "value is 34.4285714286, expected 24": 8, + "value is 28.6666666667, expected 24": 7, + "value is 180, expected 24": 7, + "value is 49.1428571429, expected 24": 7, + "value is 13.6666666667, expected 24": 7, + "value is 7.27272727273, expected 24": 7, + "value is 11.6666666667, expected 24": 7, + "value is 89.1428571429, expected 24": 7, + "value is 24.7272727273, expected 24": 7, + "value is 4.58333333333, expected 24": 7, + "value is 26.5714285714, expected 24": 7, + "value is 14.5454545455, expected 24": 7, + "value is 1197, expected 24": 7, + "value is 1044, expected 24": 7, + "value is 23.3333333333, expected 24": 6, + "value is 11.4285714286, expected 24": 6, + "value is 208, expected 24": 6, + "value is 89, expected 24": 6, + "value is 51.3333333333, expected 24": 6, + "value is 51, expected 24": 6, + "value is 1.85714285714, expected 24": 6, + "value is 67.1428571429, expected 24": 6, + "value is 73, expected 24": 6, + "value is 59, expected 24": 6, + "value is 23.7777777778, expected 24": 6, + "value is 47.2, expected 24": 6, + "value is 21.6, expected 24": 6, + "value is 20.3333333333, expected 24": 6, + "value is -56, expected 24": 6, + "value is 23.6, expected 24": 6, + "value is 20.25, expected 24": 6, + "value is 7.8, expected 24": 6, + "value is 23.6666666667, expected 24": 6, + "value is 11.5384615385, expected 24": 6, + "value is 17.4, expected 24": 6, + "value is 25.3333333333, expected 24": 6, + "value is 23.25, expected 24": 6, + "value is 17.7777777778, expected 24": 6, + "value is 35.1, expected 24": 6, + "value is 12.7272727273, expected 24": 6, + "value is 16.2, expected 24": 6, + "value is 22.25, expected 24": 6, + "value is 11.8333333333, expected 24": 6, + "value is 354, expected 24": 6, + "value is 19.8333333333, expected 24": 6, + "value is 252, expected 24": 5, + "value is 92, expected 24": 5, + "value is 9.42857142857, expected 24": 5, + "value is 19.3846153846, expected 24": 5, + "value is -28, expected 24": 5, + "value is 22.4285714286, expected 24": 5, + "value is 45.3333333333, expected 24": 5, + "value is 35.2, expected 24": 5, + "value is 22.2, expected 24": 5, + "value is 22.6, expected 24": 5, + "value is 21.8333333333, expected 24": 5, + "value is 24.8333333333, expected 24": 5, + "value is 33.6666666667, expected 24": 5, + "value is 40.4285714286, expected 24": 5, + "value is 18.8571428571, expected 24": 5, + "value is 7.33333333333, expected 24": 5, + "value is 9.81818181818, expected 24": 5, + "value is 38.6666666667, expected 24": 5, + "value is 130, expected 24": 5, + "value is 15.75, expected 24": 5, + "value is 14.6666666667, expected 24": 5, + "value is 16.3636363636, expected 24": 5, + "value is 5.83333333333, expected 24": 5, + "value is 9.83333333333, expected 24": 5, + "value is 16.25, expected 24": 5, + "value is 37.5, expected 24": 4, + "value is 27.5, expected 24": 4, + "value is 258, expected 24": 4, + "value is 34.6666666667, expected 24": 4, + "value is 37.6, expected 24": 4, + "value is 13.3846153846, expected 24": 4, + "value is 69, expected 24": 4, + "value is 10.6363636364, expected 24": 4, + "value is 30.5, expected 24": 4, + "value is 8.5, expected 24": 4, + "value is 15.5, expected 24": 4, + "value is 320, expected 24": 4, + "value is 19.2, expected 24": 4, + "value is -42, expected 24": 4, + "value is 83, expected 24": 4, + "value is 22.75, expected 24": 4, + "value is 5.5, expected 24": 4, + "value is 357, expected 24": 4, + "value is 8.61538461538, expected 24": 4, + "value is 344, expected 24": 4, + "value is 19.8, expected 24": 4, + "value is 23.5, expected 24": 4, + "value is 40.8, expected 24": 4, + "value is 25.0909090909, expected 24": 4, + "value is 27.4, expected 24": 4, + "value is 32.4, expected 24": 4, + "value is 15.25, expected 24": 4, + "value is 76, expected 24": 4, + "value is 845, expected 24": 4, + "value is 28.6, expected 24": 4, + "value is 25.7142857143, expected 24": 4, + "value is -9.6, expected 24": 4, + "value is 243, expected 24": 4, + "value is 39.1111111111, expected 24": 4, + "value is 12.8, expected 24": 4, + "value is -48, expected 24": 4, + "value is 17.4545454545, expected 24": 4, + "value is 5.2, expected 24": 4, + "value is 19.1111111111, expected 24": 4, + "value is 176, expected 24": 4, + "value is -91, expected 24": 4, + "value is 22.7777777778, expected 24": 4, + "value is 1080, expected 24": 4, + "value is -108, expected 24": 4, + "value is 111, expected 24": 4, + "value is 174, expected 24": 3, + "value is 216, expected 24": 3, + "value is 4.33333333333, expected 24": 3, + "value is 16.4444444444, expected 24": 3, + "value is 100, expected 24": 3, + "value is 19.6, expected 24": 3, + "value is 59.5, expected 24": 3, + "value is 12.6, expected 24": 3, + "value is 7.5, expected 24": 3, + "value is 7.38461538462, expected 24": 3, + "value is -4, expected 24": 3, + "value is 29.7142857143, expected 24": 3, + "value is 48.7272727273, expected 24": 3, + "value is 72.2857142857, expected 24": 3, + "value is 48.3, expected 24": 3, + "value is 5.25, expected 24": 3, + "value is 22.7692307692, expected 24": 3, + "value is 47.6666666667, expected 24": 3, + "value is 22.1666666667, expected 24": 3, + "value is 19.6363636364, expected 24": 3, + "value is 74.7692307692, expected 24": 3, + "value is 68.4, expected 24": 3, + "value is 4.14285714286, expected 24": 3, + "value is 4.91666666667, expected 24": 3, + "value is -66, expected 24": 3, + "value is 22.8333333333, expected 24": 3, + "value is 6.46153846154, expected 24": 3, + "value is 23.1428571429, expected 24": 3, + "value is 36.3636363636, expected 24": 3, + "value is 1, expected 24": 3, + "value is 154, expected 24": 3, + "value is 29.25, expected 24": 3, + "value is -21.3333333333, expected 24": 3, + "value is 480, expected 24": 3, + "value is 8.16666666667, expected 24": 3, + "value is 36.4, expected 24": 3, + "value is 106, expected 24": 3, + "value is -1.11340206186, expected 24": 3, + "value is 19.25, expected 24": 3, + "value is 9.25, expected 24": 3, + "value is 8.08333333333, expected 24": 3, + "value is 17.2222222222, expected 24": 3, + "value is 156, expected 24": 2, + "value is 31.2, expected 24": 2, + "value is 4.85714285714, expected 24": 2, + "value is 21.2, expected 24": 2, + "value is 10.3846153846, expected 24": 2, + "value is -10, expected 24": 2, + "value is 5.45454545455, expected 24": 2, + "value is 32.8, expected 24": 2, + "value is 2.18181818182, expected 24": 2, + "value is -2.18181818182, expected 24": 2, + "value is 11.1428571429, expected 24": 2, + "value is 33.4285714286, expected 24": 2, + "value is 33.6, expected 24": 2, + "value is 22.8, expected 24": 2, + "value is 8.8, expected 24": 2, + "value is 2.75, expected 24": 2, + "value is 17.875, expected 24": 2, + "value is 34.4615384615, expected 24": 2, + "value is 22.5714285714, expected 24": 2, + "value is 46.6666666667, expected 24": 2, + "value is 8.72727272727, expected 24": 2, + "value is 15.6666666667, expected 24": 2, + "value is 135, expected 24": 2, + "value is 50.5, expected 24": 2, + "value is 29.5, expected 24": 2, + "value is -72, expected 24": 2, + "value is 28.1666666667, expected 24": 2, + "value is -165, expected 24": 2, + "value is 429, expected 24": 2, + "value is 18.5, expected 24": 2, + "value is 30.1428571429, expected 24": 2, + "value is 40.8571428571, expected 24": 2, + "value is 15.2, expected 24": 2, + "value is -5, expected 24": 2, + "value is 5.53846153846, expected 24": 2, + "value is 6.07692307692, expected 24": 2, + "value is 148, expected 24": 2, + "value is 8.09090909091, expected 24": 2, + "value is 23.8333333333, expected 24": 2, + "value is 42.3076923077, expected 24": 2, + "value is 1.28571428571, expected 24": 2, + "value is 264, expected 24": 2, + "value is 62.4, expected 24": 2, + "value is 38.4, expected 24": 2, + "value is 33.5, expected 24": 2, + "value is 24.6666666667, expected 24": 2, + "value is 22.6153846154, expected 24": 2, + "value is 15.1666666667, expected 24": 2, + "value is 9.75, expected 24": 2, + "value is 15.3333333333, expected 24": 2, + "value is 13.6363636364, expected 24": 2, + "value is 6.92307692308, expected 24": 2, + "value is 175, expected 24": 2, + "value is 4.25, expected 24": 2, + "value is 53.8888888889, expected 24": 2, + "value is 41.8181818182, expected 24": 2, + "value is 24.4, expected 24": 2, + "value is 116, expected 24": 2, + "value is 7.42857142857, expected 24": 2, + "value is -4.5, expected 24": 2, + "value is 8.75, expected 24": 2, + "value is 26.8571428571, expected 24": 2, + "value is 8.44444444444, expected 24": 2, + "value is 66.6666666667, expected 24": 2, + "value is -2.57142857143, expected 24": 2, + "value is 34.5555555556, expected 24": 2, + "value is 299, expected 24": 2, + "value is 29.75, expected 24": 2, + "value is -21, expected 24": 2, + "value is 14.75, expected 24": 2, + "value is 52.5, expected 24": 2, + "value is 17.5, expected 24": 2, + "value is 11.3333333333, expected 24": 2, + "value is 24.4615384615, expected 24": 2, + "value is 25.3846153846, expected 24": 2, + "value is 23.4545454545, expected 24": 2, + "value is 20.7777777778, expected 24": 2, + "value is 15.5454545455, expected 24": 2, + "value is -1, expected 24": 2, + "value is 33.2, expected 24": 2, + "value is 7.58333333333, expected 24": 2, + "value is 58.1538461538, expected 24": 2, + "value is 10.9090909091, expected 24": 2, + "value is 1.24137931034, expected 24": 2, + "value is 14.5, expected 24": 1, + "value is 125, expected 24": 1, + "value is 12.2857142857, expected 24": 1, + "value is 151.666666667, expected 24": 1, + "value is 10.5714285714, expected 24": 1, + "value is 18.3076923077, expected 24": 1, + "value is 16.7777777778, expected 24": 1, + "value is -45.5, expected 24": 1, + "value is 221, expected 24": 1, + "value is 312, expected 24": 1, + "value is 151, expected 24": 1, + "value is 2.69230769231, expected 24": 1, + "value is 37.5555555556, expected 24": 1, + "value is 1.30769230769, expected 24": 1, + "value is 37.7142857143, expected 24": 1, + "value is 85, expected 24": 1, + "value is 440, expected 24": 1, + "value is 23.4285714286, expected 24": 1, + "value is 6.3, expected 24": 1, + "value is -4.14285714286, expected 24": 1, + "value is 66.5, expected 24": 1, + "value is 4.90909090909, expected 24": 1, + "value is 19.4, expected 24": 1, + "value is 25.2, expected 24": 1, + "value is -49, expected 24": 1, + "value is 115.909090909, expected 24": 1, + "value is -2.69230769231, expected 24": 1, + "value is 24.1818181818, expected 24": 1, + "value is -84, expected 24": 1, + "value is 74, expected 24": 1, + "value is 53.3333333333, expected 24": 1, + "value is 62.1428571429, expected 24": 1, + "value is 0.674698795181, expected 24": 1, + "value is 54.375, expected 24": 1, + "value is 7.28571428571, expected 24": 1, + "value is 94.8571428571, expected 24": 1, + "value is 107, expected 24": 1, + "value is 11.9230769231, expected 24": 1, + "value is -96, expected 24": 1, + "value is -5.375, expected 24": 1, + "value is 12.3076923077, expected 24": 1, + "value is 95, expected 24": 1, + "value is -27, expected 24": 1, + "value is 242, expected 24": 1, + "value is 5.33333333333, expected 24": 1, + "value is -3.5, expected 24": 1, + "value is 79.3333333333, expected 24": 1, + "value is -54, expected 24": 1, + "value is 594, expected 24": 1, + "value is 5.625, expected 24": 1, + "value is 184, expected 24": 1, + "value is 4.125, expected 24": 1, + "value is 21.8, expected 24": 1, + "value is -171, expected 24": 1, + "value is 35.8333333333, expected 24": 1, + "value is 51.2, expected 24": 1, + "value is -243, expected 24": 1, + "value is 29.8, expected 24": 1, + "value is 53.5714285714, expected 24": 1, + "value is 8.57142857143, expected 24": 1, + "value is 13.0909090909, expected 24": 1, + "value is 350, expected 24": 1, + "value is 26.8, expected 24": 1, + "value is 17.2, expected 24": 1, + "value is 94.4, expected 24": 1, + "value is 256, expected 24": 1, + "value is 32.25, expected 24": 1, + "value is 5.85714285714, expected 24": 1, + "value is -221, expected 24": 1, + "value is 9.30769230769, expected 24": 1, + "value is 2.15384615385, expected 24": 1, + "value is 30.5714285714, expected 24": 1, + "value is 24.7142857143, expected 24": 1, + "value is 17.5384615385, expected 24": 1, + "value is 16.6153846154, expected 24": 1, + "value is 11.0769230769, expected 24": 1, + "value is 27.25, expected 24": 1, + "value is -32, expected 24": 1, + "value is 21.25, expected 24": 1, + "value is 6.25, expected 24": 1, + "value is 9.375, expected 24": 1, + "value is 137, expected 24": 1, + "value is 11.4615384615, expected 24": 1, + "value is 143, expected 24": 1, + "value is 924, expected 24": 1, + "value is 149, expected 24": 1, + "value is 6.09090909091, expected 24": 1, + "value is 16.0769230769, expected 24": 1, + "value is 209, expected 24": 1, + "value is 1.33333333333, expected 24": 1, + "value is 792, expected 24": 1, + "value is 871, expected 24": 1, + "value is 7.69230769231, expected 24": 1, + "value is 43.5, expected 24": 1, + "value is 6.16666666667, expected 24": 1, + "value is 25.8, expected 24": 1, + "value is 18.4285714286, expected 24": 1, + "value is 21.8571428571, expected 24": 1, + "value is 16.2222222222, expected 24": 1, + "value is 24.6, expected 24": 1, + "value is 4.4, expected 24": 1, + "value is 1.45454545455, expected 24": 1, + "value is 220, expected 24": 1, + "value is 300, expected 24": 1, + "value is 19.6666666667, expected 24": 1, + "value is 27.3333333333, expected 24": 1, + "value is 46.2857142857, expected 24": 1, + "value is -50, expected 24": 1, + "value is 4.44444444444, expected 24": 1, + "value is 94, expected 24": 1, + "value is 48.9, expected 24": 1, + "value is 97.8, expected 24": 1, + "value is 162, expected 24": 1, + "value is 247, expected 24": 1, + "value is 29.6666666667, expected 24": 1, + "value is 217, expected 24": 1, + "value is 64.1666666667, expected 24": 1, + "value is 20.4, expected 24": 1, + "value is -26.6666666667, expected 24": 1, + "value is 36.8181818182, expected 24": 1, + "value is 45.1, expected 24": 1, + "value is 24.25, expected 24": 1, + "value is 9.77777777778, expected 24": 1, + "unmatched ')' (, line 1)": 1, + "value is 23.625, expected 24": 1, + "value is -2.12903225806, expected 24": 1, + "value is 141.166666667, expected 24": 1, + "value is 80.5714285714, expected 24": 1, + "value is 54.6666666667, expected 24": 1, + "value is -14.4, expected 24": 1, + "value is 74.6666666667, expected 24": 1, + "value is 29.1428571429, expected 24": 1, + "value is 61.9090909091, expected 24": 1, + "value is 4.42857142857, expected 24": 1, + "value is 74.1818181818, expected 24": 1, + "value is 26.75, expected 24": 1, + "value is -52, expected 24": 1, + "value is 2.6, expected 24": 1, + "value is 19.4285714286, expected 24": 1, + "value is -2.55555555556, expected 24": 1, + "value is 348, expected 24": 1, + "value is 3.42857142857, expected 24": 1, + "value is 23.75, expected 24": 1, + "value is 27.75, expected 24": 1, + "value is 6.6, expected 24": 1, + "value is 2.4, expected 24": 1, + "value is 0.96, expected 24": 1, + "value is 192, expected 24": 1, + "value is 62.6666666667, expected 24": 1, + "value is 124, expected 24": 1, + "value is 17.1111111111, expected 24": 1, + "value is 20.2222222222, expected 24": 1, + "value is 19.4444444444, expected 24": 1, + "value is 38.7272727273, expected 24": 1, + "value is 102, expected 24": 1, + "value is 18.8333333333, expected 24": 1, + "value is 12.1666666667, expected 24": 1, + "value is 318, expected 24": 1, + "value is 3.57142857143, expected 24": 1, + "value is 21.4444444444, expected 24": 1, + "value is 107.555555556, expected 24": 1, + "value is 5.54545454545, expected 24": 1, + "value is 119.875, expected 24": 1, + "value is 896, expected 24": 1, + "value is 12.5555555556, expected 24": 1, + "value is 14.3333333333, expected 24": 1, + "value is 20.875, expected 24": 1, + "value is 1.375, expected 24": 1, + "value is 880, expected 24": 1, + "value is 86.7777777778, expected 24": 1, + "'(' was never closed (, line 1)": 1, + "value is 8.81818181818, expected 24": 1, + "value is 62.5, expected 24": 1, + "value is 80.5, expected 24": 1, + "value is 5.38461538462, expected 24": 1, + "value is 32.5, expected 24": 1, + "value is 1.05405405405, expected 24": 1, + "value is 81.5, expected 24": 1, + "value is 46.8, expected 24": 1, + "value is 61.2, expected 24": 1, + "value is 17.25, expected 24": 1, + "value is -1.51724137931, expected 24": 1, + "value is 1296, expected 24": 1, + "unsupported expression node: BinOp": 1, + "value is -44, expected 24": 1, + "value is -0.123711340206, expected 24": 1, + "value is 129, expected 24": 1, + "value is 288, expected 24": 1, + "value is 396, expected 24": 1, + "value is -1.24137931034, expected 24": 1, + "value is 11.75, expected 24": 1 + } +} diff --git a/results/countdown_trained_greedy_summary.json b/results/countdown_trained_greedy_summary.json new file mode 100644 index 0000000..44dd9f9 --- /dev/null +++ b/results/countdown_trained_greedy_summary.json @@ -0,0 +1,107 @@ +{ + "count": 100, + "solved": 1, + "solved_rate": 0.01, + "solved_at_n_count": 1, + "solved_at_n_rate": 0.01, + "pass_at_1_count": 1, + "pass_at_1_rate": 0.01, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "numbers are not used exactly once": 32, + "value is 222, expected 88": 1, + "ok": 1, + "value is 830, expected 77": 1, + "value is 17.4025974026, expected 44": 1, + "value is 272, expected 17": 1, + "value is 44.9770114943, expected 96": 1, + "value is 36, expected 20": 1, + "value is 99, expected 30": 1, + "value is 26.5, expected 76": 1, + "value is 94.08, expected 24": 1, + "value is 270, expected 90": 1, + "value is 934, expected 48": 1, + "value is 72.8, expected 100": 1, + "value is 1216, expected 17": 1, + "value is 30, expected 26": 1, + "value is 23.0322580645, expected 44": 1, + "value is 40.2906976744, expected 52": 1, + "value is 61, expected 75": 1, + "value is 495, expected 28": 1, + "value is 1664, expected 96": 1, + "value is 32, expected 36": 1, + "value is 156, expected 98": 1, + "value is 1320, expected 92": 1, + "value is 241, expected 94": 1, + "value is 63, expected 28": 1, + "value is 16.2608695652, expected 33": 1, + "value is 715, expected 78": 1, + "numbers must be positive integers": 1, + "value is 624, expected 83": 1, + "value is 642, expected 53": 1, + "value is 56, expected 22": 1, + "value is 924, expected 65": 1, + "value is 350, expected 26": 1, + "value is 225, expected 50": 1, + "value is 960, expected 80": 1, + "value is 15.5833333333, expected 38": 1, + "value is 107.666666667, expected 61": 1, + "value is 129, expected 65": 1, + "value is 308, expected 58": 1, + "value is 102, expected 37": 1, + "value is 99, expected 100": 1, + "value is 54.3396226415, expected 36": 1, + "value is 501, expected 83": 1, + "value is 259, expected 47": 1, + "value is 120, expected 32": 1, + "value is 496, expected 100": 1, + "value is 840, expected 62": 1, + "value is 310, expected 41": 1, + "value is 390, expected 43": 1, + "value is 14, expected 15": 1, + "value is 28.5185185185, expected 71": 1, + "value is 131, expected 14": 1, + "value is 42, expected 30": 1, + "value is 9.35, expected 24": 1, + "value is 52.1276595745, expected 26": 1, + "value is 88, expected 26": 1, + "value is 143, expected 35": 1, + "value is 777, expected 59": 1, + "value is 2494, expected 66": 1, + "value is 992, expected 60": 1, + "value is 440, expected 15": 1, + "value is 38.8153846154, expected 83": 1, + "value is 192.705882353, expected 53": 1, + "value is 96, expected 78": 1, + "value is 727, expected 87": 1, + "value is 32.0641025641, expected 58": 1, + "value is 12.253164557, expected 51": 1, + "value is 426, expected 92": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 1.0, + "first_attempt_answer_tag_rate": 1.0, + "first_attempt_allowed_chars_rate": 1.0, + "first_attempt_parse_ok_rate": 0.99, + "first_attempt_numbers_ok_rate": 0.67, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.01, + "attempt_count": 100, + "attempt_r1_format_rate": 1.0, + "attempt_answer_tag_rate": 1.0, + "attempt_allowed_chars_rate": 1.0, + "attempt_parse_ok_rate": 0.99, + "attempt_numbers_ok_rate": 0.67, + "attempt_target_ok_rate": 0.01, + "attempt_valid_rate": 0.01, + "mean_candidates_until_solution": 1, + "median_candidates_until_solution": 1, + "max_candidates_until_solution": 1, + "solved_at_1_count": 1, + "solved_at_1": 0.01, + "num_samples": 1 +} diff --git a/results/countdown_trained_sample16_summary.json b/results/countdown_trained_sample16_summary.json new file mode 100644 index 0000000..05853e9 --- /dev/null +++ b/results/countdown_trained_sample16_summary.json @@ -0,0 +1,587 @@ +{ + "count": 100, + "solved": 18, + "solved_rate": 0.18, + "solved_at_n_count": 18, + "solved_at_n_rate": 0.18, + "pass_at_1_count": 1, + "pass_at_1_rate": 0.01, + "candidate_count_total": 1600, + "candidate_count_mean_per_problem": 16, + "num_samples_max": 16, + "num_samples_min": 16, + "reason_counts": { + "numbers are not used exactly once": 614, + "ok": 31, + "numbers must be positive integers": 26, + "contains disallowed characters": 10, + "value is 840, expected 62": 10, + "value is 88, expected 26": 10, + "value is 36, expected 20": 9, + "value is 16.2608695652, expected 33": 9, + "empty expression": 8, + "value is 222, expected 88": 7, + "value is 496, expected 100": 7, + "value is 777, expected 59": 7, + "unsupported expression node: UnaryOp": 6, + "value is 23.0322580645, expected 44": 6, + "value is 350, expected 26": 6, + "value is 1600, expected 100": 6, + "value is 120, expected 32": 6, + "value is 310, expected 41": 6, + "value is 992, expected 60": 6, + "value is 830, expected 77": 5, + "value is 19.4520547945, expected 44": 5, + "value is 99, expected 30": 5, + "value is 3572, expected 97": 5, + "value is 79, expected 17": 5, + "value is 1664, expected 96": 5, + "value is 715, expected 78": 5, + "value is 56, expected 22": 5, + "value is 21, expected 38": 5, + "value is 102, expected 37": 5, + "value is 392, expected 45": 5, + "value is 15, expected 66": 5, + "value is 133, expected 77": 5, + "value is 483, expected 62": 4, + "value is 106, expected 76": 4, + "value is 94.08, expected 24": 4, + "value is 47.2131147541, expected 90": 4, + "value is 895, expected 75": 4, + "value is 32, expected 36": 4, + "value is 924, expected 65": 4, + "value is 960, expected 80": 4, + "value is 218.75, expected 61": 4, + "value is 129, expected 65": 4, + "value is 36, expected 37": 4, + "value is 259, expected 47": 4, + "value is 798, expected 43": 4, + "value is 14, expected 15": 4, + "value is 393, expected 71": 4, + "value is 324, expected 36": 4, + "value is 131, expected 14": 4, + "value is 638, expected 83": 4, + "value is 727, expected 87": 4, + "value is 300, expected 87": 4, + "value is 1900, expected 51": 4, + "value is 324, expected 92": 4, + "value is 17.4025974026, expected 44": 3, + "value is 272, expected 17": 3, + "value is 44.9770114943, expected 96": 3, + "value is 27, expected 28": 3, + "value is 3956, expected 76": 3, + "value is 74.170212766, expected 48": 3, + "value is 595, expected 100": 3, + "value is 703, expected 97": 3, + "value is 1216, expected 17": 3, + "value is 30, expected 26": 3, + "value is 3.42857142857, expected 52": 3, + "value is 61, expected 75": 3, + "value is 1596, expected 75": 3, + "value is 1943, expected 96": 3, + "value is 156, expected 98": 3, + "value is 888, expected 13": 3, + "value is 241, expected 94": 3, + "value is 675, expected 83": 3, + "value is 96.28, expected 53": 3, + "value is 99, expected 80": 3, + "value is 243, expected 65": 3, + "value is 255.461538462, expected 65": 3, + "value is 26.6666666667, expected 58": 3, + "value is 501, expected 83": 3, + "value is 437, expected 45": 3, + "value is 390, expected 43": 3, + "value is 2255, expected 96": 3, + "value is 144, expected 14": 3, + "value is 359, expected 14": 3, + "value is 21.0526315789, expected 24": 3, + "value is 143, expected 35": 3, + "value is 129, expected 35": 3, + "value is 154, expected 66": 3, + "value is 371, expected 60": 3, + "value is 440, expected 15": 3, + "value is 154, expected 29": 3, + "value is 68, expected 22": 3, + "value is 153, expected 53": 3, + "value is 96, expected 78": 3, + "value is 11.7049180328, expected 58": 3, + "value is 46.6363636364, expected 51": 3, + "value is 426, expected 92": 3, + "value is 964, expected 77": 2, + "value is 4.12987012987, expected 44": 2, + "value is 11658, expected 96": 2, + "value is 139, expected 28": 2, + "value is 26.5, expected 76": 2, + "value is 93.4375, expected 24": 2, + "value is 155.52, expected 24": 2, + "value is 46.8, expected 90": 2, + "value is 92, expected 48": 2, + "value is 14774, expected 48": 2, + "value is -8.93617021277, expected 100": 2, + "value is 505, expected 100": 2, + "value is 1748, expected 44": 2, + "value is 20.8387096774, expected 44": 2, + "value is 186, expected 28": 2, + "value is 151, expected 98": 2, + "value is 1320, expected 92": 2, + "value is 26.1044776119, expected 92": 2, + "value is 229.210526316, expected 92": 2, + "value is 133, expected 94": 2, + "value is 224, expected 94": 2, + "value is -82, expected 94": 2, + "value is 63, expected 28": 2, + "value is 3.96907216495, expected 28": 2, + "value is 8.87628865979, expected 28": 2, + "value is 1080, expected 78": 2, + "value is 1680, expected 83": 2, + "value is 1722, expected 83": 2, + "value is 642, expected 53": 2, + "value is 1190, expected 53": 2, + "value is -225, expected 50": 2, + "value is 225, expected 50": 2, + "value is 819, expected 50": 2, + "value is 131.2, expected 14": 2, + "value is 73.2631578947, expected 90": 2, + "value is 268.444444444, expected 90": 2, + "value is 69.9090909091, expected 38": 2, + "value is 1857.77777778, expected 38": 2, + "value is 440, expected 61": 2, + "value is -1110, expected 65": 2, + "value is 774, expected 77": 2, + "value is 87.3658536585, expected 58": 2, + "value is 308, expected 58": 2, + "value is 170, expected 91": 2, + "value is 70, expected 100": 2, + "value is 1426, expected 100": 2, + "value is 1750, expected 36": 2, + "value is 121.411111111, expected 36": 2, + "value is 34.2, expected 36": 2, + "value is 16.7368421053, expected 83": 2, + "value is 84, expected 47": 2, + "value is 111, expected 47": 2, + "value is 108, expected 59": 2, + "value is 99, expected 59": 2, + "value is 1128, expected 15": 2, + "value is 1209, expected 15": 2, + "value is 2229, expected 71": 2, + "value is 722, expected 57": 2, + "value is 59.5833333333, expected 30": 2, + "value is 571, expected 30": 2, + "value is 27.3962264151, expected 30": 2, + "value is 782, expected 11": 2, + "value is 4480, expected 24": 2, + "value is 59840, expected 24": 2, + "value is 9.35, expected 24": 2, + "value is 52.1276595745, expected 26": 2, + "value is -560, expected 26": 2, + "value is 28.085106383, expected 26": 2, + "value is 32.1489361702, expected 26": 2, + "value is 116, expected 35": 2, + "value is 85, expected 86": 2, + "value is 1081, expected 22": 2, + "value is 121.268656716, expected 83": 2, + "value is 1798, expected 83": 2, + "value is 630, expected 72": 2, + "value is 685.076923077, expected 72": 2, + "value is 211, expected 78": 2, + "value is 175.058823529, expected 78": 2, + "value is 5451, expected 87": 2, + "value is 9.42391304348, expected 58": 2, + "value is 52.4262295082, expected 58": 2, + "value is 5.61224489796, expected 51": 2, + "value is -32, expected 88": 1, + "division by zero": 1, + "value is 12.0833333333, expected 72": 1, + "value is 1072, expected 77": 1, + "value is -1410, expected 77": 1, + "value is 17.2307692308, expected 77": 1, + "value is 98.6268656716, expected 77": 1, + "value is 1276, expected 77": 1, + "value is 78.5820895522, expected 77": 1, + "value is -5, expected 77": 1, + "value is 45, expected 44": 1, + "value is 4.68493150685, expected 44": 1, + "value is 1203.33333333, expected 44": 1, + "value is 40, expected 44": 1, + "value is -13, expected 17": 1, + "value is 64, expected 17": 1, + "value is 40.5517241379, expected 96": 1, + "value is 160.034482759, expected 96": 1, + "value is 7654, expected 96": 1, + "value is 41.1098901099, expected 96": 1, + "value is 172, expected 96": 1, + "value is 4998, expected 96": 1, + "value is 129.612244898, expected 96": 1, + "value is 151.954022989, expected 96": 1, + "value is 396, expected 96": 1, + "value is -236.833333333, expected 96": 1, + "value is 4176, expected 96": 1, + "value is 224, expected 20": 1, + "value is 56.5925925926, expected 66": 1, + "value is 89, expected 66": 1, + "value is 219.333333333, expected 30": 1, + "value is 880, expected 62": 1, + "value is 6.27906976744, expected 76": 1, + "value is 30.2816901408, expected 76": 1, + "value is 132, expected 76": 1, + "value is 590, expected 76": 1, + "value is 1904, expected 76": 1, + "value is 182, expected 76": 1, + "value is 23.8604651163, expected 76": 1, + "value is 1152, expected 84": 1, + "value is 6429, expected 84": 1, + "value is 632.888888889, expected 55": 1, + "value is -208, expected 55": 1, + "value is 163, expected 55": 1, + "value is -260, expected 55": 1, + "value is 665, expected 55": 1, + "value is 243, expected 55": 1, + "value is 12.3076923077, expected 24": 1, + "value is 166.4, expected 24": 1, + "value is 33.5208333333, expected 24": 1, + "value is 5.66153846154, expected 24": 1, + "value is -1098, expected 90": 1, + "value is 270, expected 90": 1, + "value is 12.4590163934, expected 90": 1, + "value is -90, expected 90": 1, + "value is 928, expected 90": 1, + "value is 2897.5, expected 90": 1, + "value is 115.9, expected 90": 1, + "value is 14960, expected 48": 1, + "value is 934, expected 48": 1, + "value is 96, expected 48": 1, + "value is 186, expected 48": 1, + "value is 90.8117647059, expected 48": 1, + "value is 95.1325301205, expected 48": 1, + "value is 1020, expected 48": 1, + "value is 74.8363636364, expected 100": 1, + "value is 856, expected 100": 1, + "value is 4061, expected 100": 1, + "value is 1000, expected 100": 1, + "value is 111.904761905, expected 100": 1, + "value is -934, expected 100": 1, + "value is 72.8, expected 100": 1, + "value is 989, expected 97": 1, + "value is 2304, expected 17": 1, + "value is 14.1578947368, expected 17": 1, + "value is 2356, expected 17": 1, + "value is 59.7777777778, expected 17": 1, + "value is 128.710526316, expected 17": 1, + "value is 213, expected 17": 1, + "value is 17.7631578947, expected 17": 1, + "value is 70, expected 17": 1, + "value is 84, expected 44": 1, + "value is 113.884615385, expected 44": 1, + "value is 1739, expected 44": 1, + "value is 513, expected 44": 1, + "value is 9.14285714286, expected 44": 1, + "value is 196.923076923, expected 44": 1, + "value is 1323, expected 44": 1, + "value is 448, expected 44": 1, + "value is 36, expected 26": 1, + "value is 15.2307692308, expected 26": 1, + "value is 104, expected 26": 1, + "value is -69.2941176471, expected 44": 1, + "value is 255, expected 44": 1, + "value is 192, expected 44": 1, + "value is 438, expected 44": 1, + "value is 595, expected 44": 1, + "value is 54, expected 52": 1, + "value is 120.545454545, expected 52": 1, + "value is -36, expected 52": 1, + "value is 36, expected 52": 1, + "value is 6418.72727273, expected 52": 1, + "value is 59.476744186, expected 52": 1, + "value is 2753.1025641, expected 52": 1, + "value is 880, expected 52": 1, + "value is 53, expected 52": 1, + "value is 152.393939394, expected 52": 1, + "value is 899, expected 75": 1, + "value is 64.8, expected 75": 1, + "value is 395, expected 75": 1, + "value is -7, expected 75": 1, + "value is -210, expected 28": 1, + "value is 67, expected 28": 1, + "value is 128, expected 28": 1, + "value is 576, expected 28": 1, + "value is 6.11111111111, expected 28": 1, + "value is 276, expected 28": 1, + "value is 192, expected 28": 1, + "value is -366, expected 28": 1, + "value is 8.10752688172, expected 96": 1, + "value is 279, expected 96": 1, + "value is 184, expected 98": 1, + "value is 2013, expected 98": 1, + "value is 56, expected 98": 1, + "value is 406, expected 98": 1, + "value is 48, expected 98": 1, + "value is 2822, expected 63": 1, + "value is 123.5, expected 13": 1, + "value is 76.6153846154, expected 13": 1, + "value is 31.4776119403, expected 92": 1, + "value is 1512, expected 92": 1, + "value is 262, expected 92": 1, + "value is 73, expected 92": 1, + "value is -5313, expected 92": 1, + "value is 7.01754385965, expected 92": 1, + "value is 5300, expected 92": 1, + "value is 318.142857143, expected 94": 1, + "value is 1981, expected 94": 1, + "value is 2040, expected 94": 1, + "value is -70.2857142857, expected 94": 1, + "value is 2255, expected 94": 1, + "value is 492, expected 94": 1, + "value is 1009.5, expected 28": 1, + "value is 294, expected 28": 1, + "value is 588, expected 28": 1, + "value is 19.8453608247, expected 28": 1, + "value is 1064, expected 28": 1, + "value is 1655.92857143, expected 28": 1, + "value is 7.44117647059, expected 33": 1, + "value is 2714, expected 33": 1, + "value is 760, expected 33": 1, + "value is 702, expected 33": 1, + "value is 57, expected 24": 1, + "value is 2.12903225806, expected 24": 1, + "value is 1763, expected 83": 1, + "value is 132, expected 83": 1, + "value is 15.9104477612, expected 83": 1, + "value is 1056, expected 83": 1, + "value is -1, expected 83": 1, + "value is 0, expected 83": 1, + "value is 26.1463414634, expected 83": 1, + "value is 161.28, expected 53": 1, + "value is 700, expected 53": 1, + "value is 138, expected 53": 1, + "value is 890.384615385, expected 53": 1, + "value is 78.8461538462, expected 53": 1, + "value is 44.7529411765, expected 65": 1, + "value is 1700, expected 65": 1, + "value is 1044, expected 50": 1, + "value is 26.3157894737, expected 14": 1, + "value is 173.684210526, expected 14": 1, + "value is 855, expected 14": 1, + "value is 441.666666667, expected 14": 1, + "value is 95, expected 14": 1, + "value is 2160, expected 14": 1, + "value is 4.5, expected 14": 1, + "value is 357.894736842, expected 14": 1, + "value is 146.4, expected 14": 1, + "value is 339, expected 80": 1, + "value is 861, expected 80": 1, + "value is 135.789473684, expected 90": 1, + "value is 1800, expected 90": 1, + "value is 296.8, expected 90": 1, + "value is 86.1578947368, expected 90": 1, + "value is 27.7894736842, expected 90": 1, + "value is 880, expected 38": 1, + "value is 18.5454545455, expected 38": 1, + "value is 65.0833333333, expected 38": 1, + "value is 1012, expected 38": 1, + "value is 456, expected 19": 1, + "value is 87.4, expected 19": 1, + "value is 626.24, expected 61": 1, + "value is 130, expected 61": 1, + "value is 107.666666667, expected 61": 1, + "value is 1650, expected 61": 1, + "value is 20.52, expected 61": 1, + "value is 91, expected 65": 1, + "value is 49.5, expected 65": 1, + "value is 11088, expected 65": 1, + "value is 252, expected 65": 1, + "value is 950, expected 77": 1, + "value is 42.7826086957, expected 58": 1, + "value is 1088, expected 58": 1, + "value is 74, expected 58": 1, + "value is 40, expected 58": 1, + "value is -0.512195121951, expected 58": 1, + "value is 1020, expected 37": 1, + "value is 1100, expected 37": 1, + "value is 33.35, expected 37": 1, + "value is 110, expected 37": 1, + "value is 3, expected 37": 1, + "value is 216, expected 37": 1, + "value is -36, expected 37": 1, + "value is 7224, expected 91": 1, + "value is 47.4, expected 100": 1, + "value is 99, expected 100": 1, + "value is 760, expected 36": 1, + "value is 37.358490566, expected 36": 1, + "value is 814, expected 36": 1, + "value is 29.4444444444, expected 36": 1, + "value is 2660, expected 36": 1, + "value is 2610, expected 36": 1, + "value is 123.2, expected 36": 1, + "value is 342, expected 36": 1, + "value is 54.3396226415, expected 36": 1, + "value is 755, expected 83": 1, + "value is 473.2, expected 83": 1, + "value is 12, expected 83": 1, + "value is 3648, expected 83": 1, + "value is 519, expected 83": 1, + "value is 321, expected 83": 1, + "value is 10.8791208791, expected 83": 1, + "value is 175, expected 47": 1, + "value is 64, expected 47": 1, + "value is 44, expected 47": 1, + "value is 42, expected 45": 1, + "value is 43, expected 32": 1, + "value is 252, expected 32": 1, + "invalid syntax (, line 1)": 1, + "value is -99, expected 59": 1, + "value is 333, expected 59": 1, + "value is 4503, expected 62": 1, + "value is 59, expected 43": 1, + "value is 196, expected 96": 1, + "value is 1584, expected 15": 1, + "value is -1584, expected 15": 1, + "value is 107.925925926, expected 15": 1, + "value is 891, expected 15": 1, + "value is 672, expected 15": 1, + "value is -67.5, expected 15": 1, + "value is 107.555555556, expected 15": 1, + "value is 18.5, expected 15": 1, + "value is 103.905405405, expected 71": 1, + "value is 37.4594594595, expected 71": 1, + "value is 95.9259259259, expected 71": 1, + "value is 3807, expected 71": 1, + "value is -26.1363636364, expected 71": 1, + "value is 407, expected 24": 1, + "value is 3.53658536585, expected 14": 1, + "value is 4717, expected 36": 1, + "value is 5041, expected 36": 1, + "value is 56, expected 29": 1, + "value is 851, expected 29": 1, + "value is 61.3142857143, expected 29": 1, + "value is 49, expected 14": 1, + "value is 86, expected 14": 1, + "value is -32, expected 14": 1, + "value is 420, expected 14": 1, + "value is 4332, expected 57": 1, + "value is -0.5, expected 57": 1, + "value is 19.6363636364, expected 30": 1, + "value is 53.803030303, expected 30": 1, + "value is 768, expected 30": 1, + "value is 249.5, expected 30": 1, + "value is 2.83636363636, expected 30": 1, + "value is 594, expected 30": 1, + "value is 34.1666666667, expected 30": 1, + "value is 78, expected 30": 1, + "value is 57, expected 11": 1, + "value is 11.6315789474, expected 24": 1, + "value is 16, expected 24": 1, + "value is 136, expected 24": 1, + "value is 83.6, expected 24": 1, + "value is 1012, expected 24": 1, + "value is 162.962962963, expected 24": 1, + "value is 2663.15789474, expected 24": 1, + "value is 47.3469387755, expected 26": 1, + "value is 2184, expected 26": 1, + "value is 1395, expected 26": 1, + "value is 56.9696969697, expected 26": 1, + "value is -56.9696969697, expected 26": 1, + "value is 1045, expected 26": 1, + "value is 61, expected 35": 1, + "value is 156, expected 35": 1, + "value is 493, expected 35": 1, + "value is 816, expected 35": 1, + "value is 90, expected 35": 1, + "value is 1496, expected 35": 1, + "value is 88, expected 35": 1, + "value is 1989, expected 35": 1, + "value is 17, expected 59": 1, + "value is 3612, expected 66": 1, + "value is -2494, expected 66": 1, + "value is 13.3661971831, expected 66": 1, + "value is -154, expected 66": 1, + "value is -2520, expected 66": 1, + "value is 9.50704225352, expected 66": 1, + "value is 2494, expected 66": 1, + "value is -2, expected 15": 1, + "value is 1534, expected 86": 1, + "value is 989, expected 22": 1, + "value is -67, expected 22": 1, + "value is 1152, expected 22": 1, + "value is 40.25, expected 22": 1, + "value is 1012, expected 22": 1, + "value is 38.8153846154, expected 83": 1, + "value is 2458, expected 83": 1, + "value is 3118, expected 83": 1, + "value is 35.0597014925, expected 83": 1, + "value is 38.7894736842, expected 83": 1, + "value is 1436, expected 83": 1, + "value is 3221.59701493, expected 83": 1, + "value is -1072, expected 83": 1, + "value is 276.705882353, expected 53": 1, + "value is 2465, expected 53": 1, + "value is 26.1071428571, expected 53": 1, + "value is 358, expected 53": 1, + "value is 9408, expected 53": 1, + "value is -162.411764706, expected 53": 1, + "value is 755, expected 53": 1, + "value is 2568, expected 53": 1, + "value is 3696, expected 53": 1, + "value is 953, expected 53": 1, + "value is 3900, expected 53": 1, + "value is 912, expected 72": 1, + "value is 164.25, expected 72": 1, + "value is 282, expected 72": 1, + "value is 611.375, expected 72": 1, + "value is -1216, expected 78": 1, + "value is 33.84375, expected 78": 1, + "value is 359, expected 78": 1, + "value is 535.783783784, expected 78": 1, + "value is 103.529411765, expected 78": 1, + "value is 857, expected 78": 1, + "value is -63, expected 87": 1, + "value is 10, expected 87": 1, + "value is 153.797468354, expected 87": 1, + "value is 229, expected 87": 1, + "value is 94.9565217391, expected 58": 1, + "value is 3059.2826087, expected 58": 1, + "value is 959, expected 58": 1, + "value is 6396, expected 58": 1, + "value is 2423, expected 58": 1, + "value is 1568, expected 58": 1, + "value is 17.902173913, expected 58": 1, + "value is -40.7213114754, expected 58": 1, + "value is 4647.96078431, expected 58": 1, + "value is 95.9090909091, expected 51": 1, + "value is 158, expected 51": 1, + "value is 452, expected 51": 1, + "value is 12.253164557, expected 51": 1, + "value is -160, expected 92": 1, + "value is 15, expected 92": 1, + "value is 7.125, expected 92": 1, + "value is 96, expected 92": 1, + "value is 11.5, expected 92": 1, + "value is 98, expected 92": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 1.0, + "first_attempt_answer_tag_rate": 1.0, + "first_attempt_allowed_chars_rate": 1.0, + "first_attempt_parse_ok_rate": 0.98, + "first_attempt_numbers_ok_rate": 0.51, + "first_attempt_target_ok_rate": 0.02, + "first_attempt_valid_rate": 0.01, + "attempt_count": 1600, + "attempt_r1_format_rate": 0.995625, + "attempt_answer_tag_rate": 0.995625, + "attempt_allowed_chars_rate": 0.98875, + "attempt_parse_ok_rate": 0.9675, + "attempt_numbers_ok_rate": 0.58375, + "attempt_target_ok_rate": 0.02625, + "attempt_valid_rate": 0.019375, + "mean_candidates_until_solution": 8.055555555555555, + "median_candidates_until_solution": 9.0, + "max_candidates_until_solution": 15, + "solved_at_1_count": 1, + "solved_at_1": 0.01, + "solved_at_4_count": 4, + "solved_at_4": 0.04, + "solved_at_16_count": 18, + "solved_at_16": 0.18, + "num_samples": 16 +} diff --git a/results/countdown_zeroshot_greedy_summary.json b/results/countdown_zeroshot_greedy_summary.json new file mode 100644 index 0000000..d6176b6 --- /dev/null +++ b/results/countdown_zeroshot_greedy_summary.json @@ -0,0 +1,106 @@ +{ + "count": 100, + "solved": 1, + "solved_rate": 0.01, + "solved_at_n_count": 1, + "solved_at_n_rate": 0.01, + "pass_at_1_count": 1, + "pass_at_1_rate": 0.01, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "numbers are not used exactly once": 32, + "numbers must be positive integers": 2, + "value is 222, expected 88": 1, + "ok": 1, + "value is 830, expected 77": 1, + "value is 17.4025974026, expected 44": 1, + "value is 272, expected 17": 1, + "value is 44.9770114943, expected 96": 1, + "value is 36, expected 20": 1, + "value is 99, expected 30": 1, + "value is 26.5, expected 76": 1, + "value is 94.08, expected 24": 1, + "value is 270, expected 90": 1, + "value is 934, expected 48": 1, + "value is 72.8, expected 100": 1, + "value is 1216, expected 17": 1, + "value is 30, expected 26": 1, + "value is 23.0322580645, expected 44": 1, + "value is 40.2906976744, expected 52": 1, + "value is 61, expected 75": 1, + "value is -210, expected 28": 1, + "value is 1664, expected 96": 1, + "value is 32, expected 36": 1, + "value is 156, expected 98": 1, + "value is 73, expected 92": 1, + "value is 241, expected 94": 1, + "value is 63, expected 28": 1, + "value is 16.2608695652, expected 33": 1, + "value is 715, expected 78": 1, + "value is 1056, expected 83": 1, + "value is 642, expected 53": 1, + "value is 56, expected 22": 1, + "value is 924, expected 65": 1, + "value is 350, expected 26": 1, + "value is 225, expected 50": 1, + "value is 960, expected 80": 1, + "value is 15.5833333333, expected 38": 1, + "value is 440, expected 61": 1, + "value is 129, expected 65": 1, + "value is 308, expected 58": 1, + "value is 102, expected 37": 1, + "value is 99, expected 100": 1, + "value is 54.3396226415, expected 36": 1, + "value is 501, expected 83": 1, + "value is 120, expected 32": 1, + "value is 496, expected 100": 1, + "value is 840, expected 62": 1, + "value is 310, expected 41": 1, + "value is 390, expected 43": 1, + "value is 14, expected 15": 1, + "value is 28.5185185185, expected 71": 1, + "value is 131, expected 14": 1, + "value is 42, expected 30": 1, + "value is 9.35, expected 24": 1, + "value is 52.1276595745, expected 26": 1, + "value is 88, expected 26": 1, + "value is 143, expected 35": 1, + "value is 777, expected 59": 1, + "value is 2494, expected 66": 1, + "value is 992, expected 60": 1, + "value is 440, expected 15": 1, + "value is 1798, expected 83": 1, + "value is 192.705882353, expected 53": 1, + "value is 96, expected 78": 1, + "value is 727, expected 87": 1, + "value is 32.0641025641, expected 58": 1, + "value is 12.253164557, expected 51": 1, + "value is 426, expected 92": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 1.0, + "first_attempt_answer_tag_rate": 1.0, + "first_attempt_allowed_chars_rate": 1.0, + "first_attempt_parse_ok_rate": 0.98, + "first_attempt_numbers_ok_rate": 0.66, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.01, + "attempt_count": 100, + "attempt_r1_format_rate": 1.0, + "attempt_answer_tag_rate": 1.0, + "attempt_allowed_chars_rate": 1.0, + "attempt_parse_ok_rate": 0.98, + "attempt_numbers_ok_rate": 0.66, + "attempt_target_ok_rate": 0.01, + "attempt_valid_rate": 0.01, + "mean_candidates_until_solution": 1, + "median_candidates_until_solution": 1, + "max_candidates_until_solution": 1, + "solved_at_1_count": 1, + "solved_at_1": 0.01, + "num_samples": 1 +} diff --git a/results/countdown_zeroshot_sample16_summary.json b/results/countdown_zeroshot_sample16_summary.json new file mode 100644 index 0000000..42892a9 --- /dev/null +++ b/results/countdown_zeroshot_sample16_summary.json @@ -0,0 +1,603 @@ +{ + "count": 100, + "solved": 11, + "solved_rate": 0.11, + "solved_at_n_count": 11, + "solved_at_n_rate": 0.11, + "pass_at_1_count": 0, + "pass_at_1_rate": 0.0, + "candidate_count_total": 1600, + "candidate_count_mean_per_problem": 16, + "num_samples_max": 16, + "num_samples_min": 16, + "reason_counts": { + "numbers are not used exactly once": 615, + "numbers must be positive integers": 32, + "empty expression": 20, + "ok": 19, + "contains disallowed characters": 13, + "value is 222, expected 88": 10, + "value is 88, expected 26": 10, + "unsupported expression node: UnaryOp": 9, + "value is 61, expected 75": 8, + "value is 32, expected 36": 8, + "value is 1664, expected 96": 7, + "value is 350, expected 26": 7, + "value is 840, expected 62": 7, + "value is 727, expected 87": 7, + "value is 99, expected 30": 6, + "value is 715, expected 78": 6, + "value is 225, expected 50": 6, + "value is 120, expected 32": 6, + "value is 992, expected 60": 6, + "value is 36, expected 20": 5, + "value is 483, expected 62": 5, + "value is 30, expected 26": 5, + "value is 895, expected 75": 5, + "value is 16.2608695652, expected 33": 5, + "value is 56, expected 22": 5, + "value is 4.5, expected 14": 5, + "value is 496, expected 100": 5, + "value is 310, expected 41": 5, + "value is 390, expected 43": 5, + "value is 192.705882353, expected 53": 5, + "value is 19.4520547945, expected 44": 4, + "value is 26.5, expected 76": 4, + "value is 270, expected 90": 4, + "value is 981, expected 17": 4, + "value is 228, expected 28": 4, + "value is 156, expected 98": 4, + "value is 960, expected 80": 4, + "value is 18.5454545455, expected 38": 4, + "value is 129, expected 65": 4, + "value is 102, expected 37": 4, + "value is 1600, expected 100": 4, + "value is 437, expected 45": 4, + "value is 1.14893617021, expected 15": 4, + "value is 324, expected 36": 4, + "value is 144, expected 14": 4, + "value is 722, expected 57": 4, + "value is 777, expected 59": 4, + "value is 830, expected 77": 3, + "value is 272, expected 17": 3, + "value is 3956, expected 76": 3, + "value is -260, expected 55": 3, + "value is 11.1538461538, expected 90": 3, + "value is 505, expected 100": 3, + "value is 72.8, expected 100": 3, + "value is 79, expected 17": 3, + "value is 279, expected 96": 3, + "value is 229.210526316, expected 92": 3, + "value is 624, expected 83": 3, + "value is -700, expected 53": 3, + "value is 1190, expected 53": 3, + "value is 819, expected 50": 3, + "value is 99, expected 80": 3, + "value is 27.7894736842, expected 90": 3, + "value is 218.75, expected 61": 3, + "value is 29.4444444444, expected 36": 3, + "value is 501, expected 83": 3, + "value is 183, expected 47": 3, + "value is 392, expected 45": 3, + "value is 798, expected 43": 3, + "value is 782, expected 11": 3, + "value is 143, expected 35": 3, + "value is 48, expected 35": 3, + "value is 15, expected 66": 3, + "value is 440, expected 15": 3, + "value is 154, expected 29": 3, + "value is 40.25, expected 22": 3, + "value is 1900, expected 51": 3, + "value is 17.2307692308, expected 77": 2, + "value is 40, expected 44": 2, + "value is 5394, expected 96": 2, + "value is 40.5517241379, expected 96": 2, + "value is 99.8979591837, expected 96": 2, + "value is 4704.21428571, expected 96": 2, + "value is 139, expected 28": 2, + "value is 590, expected 76": 2, + "value is 50.92, expected 24": 2, + "value is 155.52, expected 24": 2, + "value is -1098, expected 90": 2, + "value is 23.6888888889, expected 90": 2, + "value is 90.8117647059, expected 48": 2, + "value is 924, expected 48": 2, + "value is 95.2409638554, expected 48": 2, + "value is 3572, expected 97": 2, + "value is 19.3968253968, expected 44": 2, + "value is 20.8387096774, expected 44": 2, + "value is 23.0322580645, expected 44": 2, + "value is 50.2597402597, expected 52": 2, + "value is 119, expected 52": 2, + "value is -210, expected 28": 2, + "value is 186, expected 28": 2, + "value is -8, expected 92": 2, + "value is 73, expected 92": 2, + "value is 241, expected 94": 2, + "value is 133, expected 94": 2, + "value is 546, expected 94": 2, + "value is 63, expected 28": 2, + "value is 294, expected 28": 2, + "value is 2714, expected 33": 2, + "value is 12.1764705882, expected 33": 2, + "value is 1080, expected 78": 2, + "value is 675, expected 83": 2, + "value is 1680, expected 83": 2, + "value is 138, expected 53": 2, + "value is 23, expected 65": 2, + "value is 1700, expected 65": 2, + "value is 26.3157894737, expected 14": 2, + "value is 75, expected 38": 2, + "value is 65.0833333333, expected 38": 2, + "value is 21, expected 38": 2, + "value is 456, expected 19": 2, + "value is 1650, expected 61": 2, + "value is 20.52, expected 61": 2, + "value is 3766.5, expected 65": 2, + "value is -1110, expected 65": 2, + "value is 49.5, expected 65": 2, + "value is 308, expected 58": 2, + "value is 26.6666666667, expected 58": 2, + "value is 1088, expected 58": 2, + "value is 36, expected 37": 2, + "value is -36, expected 37": 2, + "value is 701, expected 37": 2, + "value is 168, expected 91": 2, + "value is 1426, expected 100": 2, + "value is 2790, expected 36": 2, + "value is -2.4, expected 36": 2, + "value is -2660, expected 36": 2, + "value is 64, expected 47": 2, + "value is 736, expected 100": 2, + "value is 99, expected 59": 2, + "value is 2451, expected 96": 2, + "value is 1584, expected 15": 2, + "value is 80.1382978723, expected 15": 2, + "value is 34, expected 71": 2, + "value is 851, expected 29": 2, + "value is 40, expected 14": 2, + "value is 86, expected 14": 2, + "value is 359, expected 14": 2, + "value is -722, expected 57": 2, + "value is 27.3962264151, expected 30": 2, + "value is 451, expected 30": 2, + "value is 83.4339622642, expected 30": 2, + "value is 9.35, expected 24": 2, + "value is -21.0526315789, expected 24": 2, + "value is 1045, expected 26": 2, + "value is 52.1276595745, expected 26": 2, + "value is 28.085106383, expected 26": 2, + "value is -560, expected 26": 2, + "value is 71, expected 35": 2, + "value is 2494, expected 66": 2, + "value is 154, expected 66": 2, + "value is 46, expected 60": 2, + "value is 71.875, expected 22": 2, + "value is 1710, expected 83": 2, + "value is 3118, expected 83": 2, + "value is 153, expected 53": 2, + "value is 31.5714285714, expected 53": 2, + "value is 2568, expected 53": 2, + "value is -212, expected 78": 2, + "value is 96, expected 78": 2, + "value is 300, expected 87": 2, + "value is 17.902173913, expected 58": 2, + "value is 12.253164557, expected 51": 2, + "value is 5.61224489796, expected 51": 2, + "value is 452, expected 51": 2, + "value is 3454, expected 51": 2, + "value is 15, expected 92": 2, + "value is 252, expected 92": 2, + "value is 324, expected 92": 2, + "value is 32, expected 88": 1, + "value is 1290, expected 72": 1, + "value is 67, expected 71": 1, + "value is 1276, expected 77": 1, + "value is 47.8571428571, expected 77": 1, + "value is 4154, expected 77": 1, + "value is 204, expected 77": 1, + "value is 964, expected 77": 1, + "value is 8.35714285714, expected 77": 1, + "value is 93.8, expected 77": 1, + "value is 83, expected 77": 1, + "value is -74, expected 44": 1, + "value is 1203.33333333, expected 44": 1, + "value is 80.1666666667, expected 44": 1, + "value is 17.4025974026, expected 44": 1, + "value is 10, expected 44": 1, + "value is 4.12987012987, expected 44": 1, + "value is 72, expected 17": 1, + "value is 64, expected 17": 1, + "value is -13, expected 17": 1, + "value is 236.833333333, expected 96": 1, + "value is 44.9770114943, expected 96": 1, + "value is 146.274725275, expected 96": 1, + "value is 3826, expected 96": 1, + "value is 1606, expected 96": 1, + "value is 172, expected 96": 1, + "value is 2814, expected 96": 1, + "value is 1890, expected 66": 1, + "value is 23.2222222222, expected 66": 1, + "value is 79.9259259259, expected 28": 1, + "value is 27.7209302326, expected 76": 1, + "value is 313, expected 76": 1, + "value is 44.5, expected 76": 1, + "value is 504, expected 76": 1, + "value is 1689, expected 76": 1, + "value is 73.8873239437, expected 76": 1, + "value is 3358.77777778, expected 76": 1, + "value is 163, expected 55": 1, + "value is 225, expected 55": 1, + "value is 243, expected 55": 1, + "value is 36, expected 55": 1, + "value is 144, expected 55": 1, + "value is 632.888888889, expected 55": 1, + "value is 4895, expected 55": 1, + "value is 10, expected 24": 1, + "value is 93.4375, expected 24": 1, + "value is 26.5625, expected 24": 1, + "value is 111.4375, expected 24": 1, + "value is 825, expected 24": 1, + "value is 18.2153846154, expected 24": 1, + "value is 15.32, expected 24": 1, + "value is 3078.4, expected 24": 1, + "value is 1127, expected 24": 1, + "value is 94.08, expected 24": 1, + "value is 12.3076923077, expected 24": 1, + "value is 990, expected 90": 1, + "value is 12.4590163934, expected 90": 1, + "value is 1184.75555556, expected 90": 1, + "value is 47.2131147541, expected 90": 1, + "value is 74, expected 48": 1, + "value is 14774, expected 48": 1, + "value is 96, expected 48": 1, + "value is 746, expected 48": 1, + "value is 91, expected 48": 1, + "value is 747, expected 48": 1, + "value is 935, expected 48": 1, + "value is 186, expected 48": 1, + "value is 14960, expected 48": 1, + "value is 856, expected 100": 1, + "value is 2413.45238095, expected 100": 1, + "value is 595, expected 100": 1, + "value is 8736, expected 100": 1, + "value is 20.8928571429, expected 100": 1, + "value is 3631, expected 100": 1, + "value is 132, expected 97": 1, + "value is 703, expected 97": 1, + "value is 88, expected 17": 1, + "value is 2356, expected 17": 1, + "value is 90, expected 17": 1, + "value is 1216, expected 17": 1, + "value is 126, expected 17": 1, + "value is 128.710526316, expected 17": 1, + "value is 3.68421052632, expected 17": 1, + "value is 61, expected 17": 1, + "value is 297, expected 17": 1, + "value is 1932, expected 41": 1, + "value is 6.95652173913, expected 41": 1, + "value is 631, expected 44": 1, + "value is 196.269230769, expected 44": 1, + "value is 84, expected 44": 1, + "value is 3040, expected 44": 1, + "value is 88.734375, expected 44": 1, + "value is 28.1842105263, expected 44": 1, + "value is 196.923076923, expected 44": 1, + "value is 416, expected 44": 1, + "value is 104, expected 26": 1, + "value is -176, expected 26": 1, + "value is 14, expected 26": 1, + "value is 15.2307692308, expected 26": 1, + "value is 192, expected 44": 1, + "value is 86, expected 44": 1, + "value is 15.6470588235, expected 44": 1, + "value is 24, expected 44": 1, + "value is 438, expected 44": 1, + "value is 550.263157895, expected 44": 1, + "value is 1102, expected 44": 1, + "value is 58.6774193548, expected 44": 1, + "value is 99, expected 44": 1, + "value is -119, expected 44": 1, + "value is 403.939393939, expected 52": 1, + "value is 54, expected 52": 1, + "value is 2753.1025641, expected 52": 1, + "value is 53, expected 52": 1, + "value is 70.2857142857, expected 52": 1, + "value is 39.9069767442, expected 52": 1, + "value is 375, expected 52": 1, + "value is 6418.72727273, expected 52": 1, + "value is 352, expected 52": 1, + "value is 3.42857142857, expected 52": 1, + "value is 48.5128205128, expected 52": 1, + "value is 7505, expected 75": 1, + "value is 64.8, expected 75": 1, + "value is 899, expected 75": 1, + "value is 124, expected 28": 1, + "value is -366, expected 28": 1, + "value is 192, expected 28": 1, + "value is 495, expected 28": 1, + "value is -38, expected 96": 1, + "value is 70, expected 36": 1, + "value is 44, expected 98": 1, + "value is 60, expected 98": 1, + "value is 56, expected 98": 1, + "value is 147, expected 98": 1, + "value is 77, expected 98": 1, + "value is 82.3777777778, expected 63": 1, + "value is 2822, expected 63": 1, + "value is 133.392857143, expected 63": 1, + "value is 117, expected 63": 1, + "value is 888, expected 13": 1, + "value is -1340, expected 92": 1, + "value is 402, expected 92": 1, + "value is -6963, expected 92": 1, + "value is 1320, expected 92": 1, + "value is 1328.25, expected 92": 1, + "value is 1802, expected 92": 1, + "value is 293, expected 92": 1, + "value is 97, expected 94": 1, + "value is 2255, expected 94": 1, + "value is 81.4285714286, expected 94": 1, + "value is 11.1428571429, expected 94": 1, + "value is 85, expected 94": 1, + "value is 17.1219512195, expected 94": 1, + "value is 9.07317073171, expected 94": 1, + "value is 5.33333333333, expected 94": 1, + "value is 1632, expected 94": 1, + "value is -75.5714285714, expected 28": 1, + "value is -294, expected 28": 1, + "value is -63, expected 28": 1, + "value is 31.6909090909, expected 28": 1, + "value is 235.571428571, expected 28": 1, + "value is 8.87628865979, expected 28": 1, + "value is 652, expected 28": 1, + "value is 189.380952381, expected 28": 1, + "value is 3070.84057971, expected 33": 1, + "value is 3634, expected 33": 1, + "value is 102.492753623, expected 33": 1, + "value is 7.44117647059, expected 33": 1, + "value is 125.52173913, expected 33": 1, + "value is 2622, expected 33": 1, + "value is 101.076923077, expected 78": 1, + "value is 68, expected 78": 1, + "value is 217, expected 24": 1, + "value is 2.12903225806, expected 24": 1, + "value is 650, expected 83": 1, + "value is -1680, expected 83": 1, + "value is -1764, expected 83": 1, + "value is 2592, expected 83": 1, + "value is 4355, expected 83": 1, + "value is 1088, expected 83": 1, + "value is 21.32, expected 53": 1, + "value is 117.6, expected 53": 1, + "value is 4482, expected 53": 1, + "value is 87, expected 53": 1, + "value is 808, expected 53": 1, + "value is 182.6, expected 53": 1, + "value is 123.636363636, expected 65": 1, + "value is 2624, expected 65": 1, + "value is 25, expected 26": 1, + "value is 45, expected 26": 1, + "value is -225, expected 50": 1, + "value is 76, expected 50": 1, + "value is 2255, expected 14": 1, + "value is 58.6666666667, expected 14": 1, + "value is 131.2, expected 14": 1, + "value is 456.25, expected 14": 1, + "value is -15.2, expected 14": 1, + "value is 8, expected 14": 1, + "value is 675, expected 90": 1, + "value is 39.1875, expected 90": 1, + "value is 874.736842105, expected 90": 1, + "value is 149.925925926, expected 38": 1, + "value is 1140, expected 38": 1, + "value is 700, expected 38": 1, + "value is 69.9259259259, expected 38": 1, + "value is -124.235294118, expected 38": 1, + "value is 1857.77777778, expected 38": 1, + "value is 626.24, expected 61": 1, + "value is 738, expected 61": 1, + "value is 282, expected 61": 1, + "value is 158.333333333, expected 61": 1, + "value is 107.666666667, expected 61": 1, + "value is 336, expected 65": 1, + "value is 78, expected 65": 1, + "value is 84.1538461538, expected 65": 1, + "value is 91, expected 65": 1, + "value is 89, expected 65": 1, + "value is 396, expected 58": 1, + "value is 260, expected 58": 1, + "value is 41.6666666667, expected 58": 1, + "value is 71.15, expected 37": 1, + "value is 12.5094339623, expected 37": 1, + "value is 663, expected 37": 1, + "value is 110, expected 37": 1, + "value is 88, expected 100": 1, + "value is 40.7058823529, expected 100": 1, + "value is -760, expected 36": 1, + "value is 54.3396226415, expected 36": 1, + "value is 103.055555556, expected 36": 1, + "value is 123.2, expected 36": 1, + "value is 20.4, expected 83": 1, + "value is 519, expected 83": 1, + "value is 671.333333333, expected 83": 1, + "value is 246.666666667, expected 83": 1, + "value is 99.6666666667, expected 83": 1, + "value is 12, expected 83": 1, + "value is 380, expected 83": 1, + "value is 26.3157894737, expected 83": 1, + "value is 69, expected 47": 1, + "value is 259, expected 47": 1, + "value is 84, expected 47": 1, + "value is 175, expected 47": 1, + "value is 42, expected 45": 1, + "value is -12, expected 32": 1, + "value is 84, expected 100": 1, + "value is 177, expected 59": 1, + "value is 59, expected 43": 1, + "value is 408, expected 43": 1, + "value is 100, expected 96": 1, + "value is 111, expected 15": 1, + "value is -5.74074074074, expected 15": 1, + "value is 214.5, expected 15": 1, + "value is 419, expected 15": 1, + "value is 2256, expected 15": 1, + "value is 1800, expected 15": 1, + "value is 42.9545454545, expected 71": 1, + "value is 28.5185185185, expected 71": 1, + "value is 511, expected 71": 1, + "value is 29.5540540541, expected 71": 1, + "value is -28.5185185185, expected 71": 1, + "value is 1917, expected 71": 1, + "value is 37.4594594595, expected 71": 1, + "value is 5880, expected 71": 1, + "value is 114.932432432, expected 71": 1, + "value is 17, expected 14": 1, + "value is 5041, expected 36": 1, + "value is 54.8648648649, expected 29": 1, + "value is 195, expected 14": 1, + "value is 500, expected 14": 1, + "value is 87, expected 14": 1, + "value is 13, expected 14": 1, + "value is 131, expected 14": 1, + "value is 24.0652173913, expected 14": 1, + "value is 4332, expected 57": 1, + "value is 133, expected 57": 1, + "value is 545.416666667, expected 30": 1, + "value is 53.5471698113, expected 30": 1, + "value is 59.5833333333, expected 30": 1, + "value is 34.1666666667, expected 30": 1, + "value is 49.2, expected 30": 1, + "value is 2.83636363636, expected 30": 1, + "value is 78, expected 30": 1, + "value is 76, expected 30": 1, + "value is 141, expected 11": 1, + "value is 58.1176470588, expected 24": 1, + "value is -2590, expected 24": 1, + "value is 74.25, expected 24": 1, + "value is -32.1481481481, expected 24": 1, + "value is 960, expected 24": 1, + "value is 3468, expected 24": 1, + "value is -2860, expected 24": 1, + "value is 16, expected 24": 1, + "value is 2052, expected 24": 1, + "value is -25.6296296296, expected 24": 1, + "value is 265, expected 26": 1, + "value is 2184, expected 26": 1, + "value is 32.6666666667, expected 26": 1, + "value is 560, expected 26": 1, + "value is 35.3571428571, expected 26": 1, + "value is 680, expected 26": 1, + "value is 1584, expected 35": 1, + "value is 88, expected 35": 1, + "value is 1972, expected 35": 1, + "value is 1989, expected 35": 1, + "value is 129, expected 35": 1, + "value is 61, expected 35": 1, + "value is 116, expected 35": 1, + "value is 58, expected 59": 1, + "value is -17, expected 59": 1, + "value is 390, expected 66": 1, + "value is 234.846153846, expected 66": 1, + "value is 2520, expected 66": 1, + "value is 184, expected 66": 1, + "value is 261.533333333, expected 66": 1, + "value is 9.50704225352, expected 66": 1, + "value is 101, expected 66": 1, + "value is 371, expected 60": 1, + "value is 85, expected 86": 1, + "value is 9, expected 29": 1, + "value is 36, expected 29": 1, + "value is 69.9130434783, expected 22": 1, + "value is 40, expected 22": 1, + "value is 67, expected 22": 1, + "value is 1798, expected 83": 1, + "value is 302.210526316, expected 83": 1, + "value is 212, expected 83": 1, + "value is 638, expected 83": 1, + "value is 1722, expected 83": 1, + "value is 12.4776119403, expected 83": 1, + "value is 2456.32307692, expected 83": 1, + "value is 4158, expected 83": 1, + "value is 48.3731343284, expected 83": 1, + "value is 121.268656716, expected 83": 1, + "value is 40, expected 16": 1, + "value is 3900, expected 53": 1, + "value is -1241, expected 53": 1, + "value is 12.3452380952, expected 53": 1, + "value is 1222, expected 72": 1, + "value is 630, expected 72": 1, + "value is 282, expected 72": 1, + "value is 912, expected 72": 1, + "value is 29.375, expected 72": 1, + "value is 104, expected 78": 1, + "value is 58.4210526316, expected 78": 1, + "value is 103.529411765, expected 78": 1, + "value is 695, expected 78": 1, + "value is -24.7368421053, expected 78": 1, + "value is 175.058823529, expected 78": 1, + "value is -8930, expected 78": 1, + "value is 1051, expected 78": 1, + "value is 2516, expected 78": 1, + "value is 50.7058823529, expected 78": 1, + "value is 190.842105263, expected 78": 1, + "value is 475, expected 78": 1, + "value is -551, expected 87": 1, + "value is 156.85915493, expected 87": 1, + "value is -63, expected 87": 1, + "value is 5531.95061728, expected 87": 1, + "value is 221, expected 87": 1, + "value is 229, expected 87": 1, + "value is 133, expected 77": 1, + "value is 2423, expected 58": 1, + "value is -17.7608695652, expected 58": 1, + "value is 52.4262295082, expected 58": 1, + "value is 16.7450980392, expected 58": 1, + "value is 6396, expected 58": 1, + "value is 1700, expected 58": 1, + "value is 85.5326086957, expected 58": 1, + "value is 32.0392156863, expected 58": 1, + "value is 77.0543478261, expected 58": 1, + "value is 20.2692307692, expected 58": 1, + "value is 128.336956522, expected 58": 1, + "value is 9.42391304348, expected 58": 1, + "value is 21.5306122449, expected 51": 1, + "value is 46.6363636364, expected 51": 1, + "value is 3914.09090909, expected 51": 1, + "value is 608, expected 51": 1, + "value is 76, expected 92": 1, + "value is -988, expected 92": 1, + "value is 426, expected 92": 1, + "value is 828, expected 92": 1, + "value is 7.125, expected 92": 1, + "value is 11.5, expected 92": 1, + "value is 72, expected 92": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 0.97, + "first_attempt_answer_tag_rate": 0.97, + "first_attempt_allowed_chars_rate": 0.96, + "first_attempt_parse_ok_rate": 0.95, + "first_attempt_numbers_ok_rate": 0.58, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.0, + "attempt_count": 1600, + "attempt_r1_format_rate": 0.986875, + "attempt_answer_tag_rate": 0.986875, + "attempt_allowed_chars_rate": 0.979375, + "attempt_parse_ok_rate": 0.95375, + "attempt_numbers_ok_rate": 0.570625, + "attempt_target_ok_rate": 0.015625, + "attempt_valid_rate": 0.011875, + "mean_candidates_until_solution": 7.363636363636363, + "median_candidates_until_solution": 7, + "max_candidates_until_solution": 15, + "solved_at_1_count": 0, + "solved_at_1": 0.0, + "solved_at_4_count": 5, + "solved_at_4": 0.05, + "solved_at_16_count": 11, + "solved_at_16": 0.11, + "num_samples": 16 +} diff --git a/results/greedy_base_summary.json b/results/greedy_base_summary.json new file mode 100644 index 0000000..e96c0d4 --- /dev/null +++ b/results/greedy_base_summary.json @@ -0,0 +1,60 @@ +{ + "count": 100, + "solved": 0, + "solved_rate": 0.0, + "solved_at_n_count": 0, + "solved_at_n_rate": 0.0, + "pass_at_1_count": 0, + "pass_at_1_rate": 0.0, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "empty expression": 43, + "contains disallowed characters": 20, + "numbers are not used exactly once": 17, + "value is 34, expected 24": 2, + "value is 16, expected 24": 1, + "value is 40, expected 24": 1, + "value is 6, expected 24": 1, + "value is 70, expected 24": 1, + "value is 28, expected 24": 1, + "unsupported expression node: UnaryOp": 1, + "value is 5, expected 24": 1, + "value is -0.875, expected 24": 1, + "value is 25, expected 24": 1, + "value is 32, expected 24": 1, + "value is 153, expected 24": 1, + "value is 6.85714285714, expected 24": 1, + "value is 89, expected 24": 1, + "value is 96, expected 24": 1, + "value is 23, expected 24": 1, + "value is 58.5, expected 24": 1, + "value is 30, expected 24": 1, + "value is 26, expected 24": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 0.65, + "first_attempt_answer_tag_rate": 0.65, + "first_attempt_allowed_chars_rate": 0.37, + "first_attempt_parse_ok_rate": 0.36, + "first_attempt_numbers_ok_rate": 0.19, + "first_attempt_target_ok_rate": 0.03, + "first_attempt_valid_rate": 0.0, + "attempt_count": 100, + "attempt_r1_format_rate": 0.65, + "attempt_answer_tag_rate": 0.65, + "attempt_allowed_chars_rate": 0.37, + "attempt_parse_ok_rate": 0.36, + "attempt_numbers_ok_rate": 0.19, + "attempt_target_ok_rate": 0.03, + "attempt_valid_rate": 0.0, + "mean_candidates_until_solution": null, + "median_candidates_until_solution": null, + "max_candidates_until_solution": null, + "solved_at_1_count": 0, + "solved_at_1": 0.0, + "num_samples": 1 +} diff --git a/results/greedy_ch_summary.json b/results/greedy_ch_summary.json new file mode 100644 index 0000000..6dc5c3a --- /dev/null +++ b/results/greedy_ch_summary.json @@ -0,0 +1,81 @@ +{ + "count": 100, + "solved": 0, + "solved_rate": 0.0, + "solved_at_n_count": 0, + "solved_at_n_rate": 0.0, + "pass_at_1_count": 0, + "pass_at_1_rate": 0.0, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "value is 20, expected 24": 6, + "value is 60, expected 24": 6, + "value is 40, expected 24": 5, + "value is 30, expected 24": 5, + "value is 72, expected 24": 5, + "value is 12, expected 24": 5, + "value is 48, expected 24": 5, + "value is 84, expected 24": 5, + "value is 36, expected 24": 5, + "value is 32, expected 24": 4, + "value is 16, expected 24": 4, + "value is 42, expected 24": 3, + "value is 18, expected 24": 3, + "value is 10, expected 24": 2, + "value is 21, expected 24": 2, + "value is 0, expected 24": 2, + "value is 96, expected 24": 2, + "value is 6, expected 24": 2, + "value is 33, expected 24": 2, + "value is 9, expected 24": 2, + "value is 25, expected 24": 2, + "value is 90, expected 24": 2, + "value is 12.5, expected 24": 1, + "value is 9.33333333333, expected 24": 1, + "value is 50, expected 24": 1, + "value is -16, expected 24": 1, + "value is 39, expected 24": 1, + "value is 6.4, expected 24": 1, + "value is 35, expected 24": 1, + "value is -136, expected 24": 1, + "value is 16.5, expected 24": 1, + "value is 3, expected 24": 1, + "value is 91, expected 24": 1, + "value is 19, expected 24": 1, + "value is -12, expected 24": 1, + "value is 26, expected 24": 1, + "value is 8, expected 24": 1, + "value is 33.3333333333, expected 24": 1, + "value is 10.5, expected 24": 1, + "value is 20.5714285714, expected 24": 1, + "value is 54, expected 24": 1, + "value is 66, expected 24": 1, + "numbers are not used exactly once": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 1.0, + "first_attempt_answer_tag_rate": 1.0, + "first_attempt_allowed_chars_rate": 1.0, + "first_attempt_parse_ok_rate": 1.0, + "first_attempt_numbers_ok_rate": 0.99, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.0, + "attempt_count": 100, + "attempt_r1_format_rate": 1.0, + "attempt_answer_tag_rate": 1.0, + "attempt_allowed_chars_rate": 1.0, + "attempt_parse_ok_rate": 1.0, + "attempt_numbers_ok_rate": 0.99, + "attempt_target_ok_rate": 0.01, + "attempt_valid_rate": 0.0, + "mean_candidates_until_solution": null, + "median_candidates_until_solution": null, + "max_candidates_until_solution": null, + "solved_at_1_count": 0, + "solved_at_1": 0.0, + "num_samples": 1 +} diff --git a/results/greedy_sft_summary.json b/results/greedy_sft_summary.json new file mode 100644 index 0000000..48fbc21 --- /dev/null +++ b/results/greedy_sft_summary.json @@ -0,0 +1,81 @@ +{ + "count": 100, + "solved": 0, + "solved_rate": 0.0, + "solved_at_n_count": 0, + "solved_at_n_rate": 0.0, + "pass_at_1_count": 0, + "pass_at_1_rate": 0.0, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "value is 48, expected 24": 6, + "value is 60, expected 24": 6, + "value is 36, expected 24": 6, + "value is 40, expected 24": 5, + "value is 30, expected 24": 5, + "value is 72, expected 24": 5, + "value is 32, expected 24": 4, + "value is 16, expected 24": 4, + "value is 12, expected 24": 4, + "value is 84, expected 24": 4, + "value is 20, expected 24": 4, + "empty expression": 3, + "value is 18, expected 24": 3, + "value is 10, expected 24": 2, + "value is 9.33333333333, expected 24": 2, + "value is 42, expected 24": 2, + "value is 21, expected 24": 2, + "value is 0, expected 24": 2, + "value is 96, expected 24": 2, + "value is 33, expected 24": 2, + "value is 9, expected 24": 2, + "value is 25, expected 24": 2, + "value is -12, expected 24": 2, + "value is 90, expected 24": 2, + "value is 13, expected 24": 1, + "value is 12.5, expected 24": 1, + "value is 50, expected 24": 1, + "value is -16, expected 24": 1, + "value is 39, expected 24": 1, + "value is 6.4, expected 24": 1, + "value is -136, expected 24": 1, + "value is 16.5, expected 24": 1, + "value is 70, expected 24": 1, + "value is 3, expected 24": 1, + "value is 91, expected 24": 1, + "value is 19, expected 24": 1, + "value is 26, expected 24": 1, + "value is 8, expected 24": 1, + "value is 33.3333333333, expected 24": 1, + "value is 14, expected 24": 1, + "value is 54, expected 24": 1, + "value is 66, expected 24": 1, + "numbers are not used exactly once": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 0.97, + "first_attempt_answer_tag_rate": 0.97, + "first_attempt_allowed_chars_rate": 0.97, + "first_attempt_parse_ok_rate": 0.97, + "first_attempt_numbers_ok_rate": 0.96, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.0, + "attempt_count": 100, + "attempt_r1_format_rate": 0.97, + "attempt_answer_tag_rate": 0.97, + "attempt_allowed_chars_rate": 0.97, + "attempt_parse_ok_rate": 0.97, + "attempt_numbers_ok_rate": 0.96, + "attempt_target_ok_rate": 0.01, + "attempt_valid_rate": 0.0, + "mean_candidates_until_solution": null, + "median_candidates_until_solution": null, + "max_candidates_until_solution": null, + "solved_at_1_count": 0, + "solved_at_1": 0.0, + "num_samples": 1 +} diff --git a/results/greedy_shaped_summary.json b/results/greedy_shaped_summary.json new file mode 100644 index 0000000..becfab6 --- /dev/null +++ b/results/greedy_shaped_summary.json @@ -0,0 +1,79 @@ +{ + "count": 100, + "solved": 0, + "solved_rate": 0.0, + "solved_at_n_count": 0, + "solved_at_n_rate": 0.0, + "pass_at_1_count": 0, + "pass_at_1_rate": 0.0, + "candidate_count_total": 100, + "candidate_count_mean_per_problem": 1, + "num_samples_max": 1, + "num_samples_min": 1, + "reason_counts": { + "value is 36, expected 24": 7, + "value is 20, expected 24": 6, + "value is 16, expected 24": 6, + "value is 60, expected 24": 6, + "value is 40, expected 24": 5, + "value is 30, expected 24": 5, + "value is 72, expected 24": 5, + "value is 48, expected 24": 5, + "value is 32, expected 24": 4, + "value is 12, expected 24": 4, + "value is 84, expected 24": 4, + "value is 42, expected 24": 3, + "value is 18, expected 24": 3, + "value is 6, expected 24": 3, + "value is 10, expected 24": 2, + "value is 21, expected 24": 2, + "value is 0, expected 24": 2, + "value is 96, expected 24": 2, + "value is 33, expected 24": 2, + "value is 9, expected 24": 2, + "value is 90, expected 24": 2, + "value is 12.5, expected 24": 1, + "value is 9.33333333333, expected 24": 1, + "value is 50, expected 24": 1, + "value is 56, expected 24": 1, + "value is -16, expected 24": 1, + "value is 39, expected 24": 1, + "value is 6.4, expected 24": 1, + "value is 91, expected 24": 1, + "value is 19, expected 24": 1, + "value is -12, expected 24": 1, + "value is 15, expected 24": 1, + "value is 26, expected 24": 1, + "value is 25, expected 24": 1, + "value is 8, expected 24": 1, + "value is 33.3333333333, expected 24": 1, + "value is 10.5, expected 24": 1, + "value is 14, expected 24": 1, + "value is 16.6666666667, expected 24": 1, + "value is 66, expected 24": 1, + "numbers are not used exactly once": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 1.0, + "first_attempt_answer_tag_rate": 1.0, + "first_attempt_allowed_chars_rate": 1.0, + "first_attempt_parse_ok_rate": 1.0, + "first_attempt_numbers_ok_rate": 0.99, + "first_attempt_target_ok_rate": 0.01, + "first_attempt_valid_rate": 0.0, + "attempt_count": 100, + "attempt_r1_format_rate": 1.0, + "attempt_answer_tag_rate": 1.0, + "attempt_allowed_chars_rate": 1.0, + "attempt_parse_ok_rate": 1.0, + "attempt_numbers_ok_rate": 0.99, + "attempt_target_ok_rate": 0.01, + "attempt_valid_rate": 0.0, + "mean_candidates_until_solution": null, + "median_candidates_until_solution": null, + "max_candidates_until_solution": null, + "solved_at_1_count": 0, + "solved_at_1": 0.0, + "num_samples": 1 +} diff --git a/results/sample16_shaped_shard0_summary.json b/results/sample16_shaped_shard0_summary.json new file mode 100644 index 0000000..de63742 --- /dev/null +++ b/results/sample16_shaped_shard0_summary.json @@ -0,0 +1,230 @@ +{ + "count": 100, + "solved": 36, + "solved_rate": 0.36, + "solved_at_n_count": 36, + "solved_at_n_rate": 0.36, + "pass_at_1_count": 5, + "pass_at_1_rate": 0.05, + "candidate_count_total": 1600, + "candidate_count_mean_per_problem": 16, + "num_samples_max": 16, + "num_samples_min": 16, + "reason_counts": { + "ok": 69, + "value is 20, expected 24": 63, + "value is 48, expected 24": 61, + "value is 18, expected 24": 57, + "value is 40, expected 24": 53, + "value is 0, expected 24": 51, + "value is 6, expected 24": 49, + "value is 60, expected 24": 48, + "value is 16, expected 24": 45, + "value is 12, expected 24": 42, + "value is 30, expected 24": 41, + "value is 84, expected 24": 41, + "value is 8, expected 24": 38, + "value is 21, expected 24": 37, + "value is 22, expected 24": 37, + "value is 36, expected 24": 37, + "value is 72, expected 24": 34, + "numbers are not used exactly once": 31, + "value is 25, expected 24": 30, + "value is 26, expected 24": 28, + "value is 28, expected 24": 26, + "empty expression": 26, + "value is 42, expected 24": 25, + "value is 32, expected 24": 24, + "value is 14, expected 24": 24, + "value is 33, expected 24": 23, + "value is 27, expected 24": 22, + "value is 10, expected 24": 19, + "value is 23, expected 24": 16, + "value is 56, expected 24": 16, + "value is 19, expected 24": 14, + "value is 96, expected 24": 14, + "value is 66, expected 24": 13, + "value is 29, expected 24": 13, + "value is 52, expected 24": 12, + "value is 54, expected 24": 12, + "value is 9, expected 24": 12, + "value is 15, expected 24": 11, + "value is 35, expected 24": 11, + "value is 50, expected 24": 11, + "value is 120, expected 24": 11, + "value is 9.33333333333, expected 24": 10, + "value is 7, expected 24": 10, + "value is 31, expected 24": 10, + "value is 17, expected 24": 9, + "value is 44, expected 24": 9, + "value is 39, expected 24": 9, + "value is 37, expected 24": 8, + "value is 13, expected 24": 8, + "value is 10.5, expected 24": 8, + "value is 18.6666666667, expected 24": 7, + "value is 34, expected 24": 7, + "value is 90, expected 24": 7, + "value is 64, expected 24": 7, + "value is -12, expected 24": 7, + "value is 12.5, expected 24": 6, + "value is 43, expected 24": 6, + "value is 4, expected 24": 6, + "value is 70, expected 24": 6, + "value is 80, expected 24": 6, + "value is 104, expected 24": 5, + "value is 45, expected 24": 5, + "value is 22.5, expected 24": 4, + "value is 49, expected 24": 4, + "value is 20.5714285714, expected 24": 4, + "value is 5, expected 24": 4, + "value is 16.6666666667, expected 24": 4, + "value is 77, expected 24": 4, + "value is 91, expected 24": 4, + "value is 108, expected 24": 4, + "value is 25.6, expected 24": 4, + "value is 25.5, expected 24": 3, + "value is 46, expected 24": 3, + "value is -7, expected 24": 3, + "value is 12.8571428571, expected 24": 3, + "value is 65, expected 24": 3, + "value is 38, expected 24": 3, + "value is 16.5, expected 24": 3, + "value is 88, expected 24": 3, + "value is -8, expected 24": 3, + "value is 24.2, expected 24": 3, + "value is -16, expected 24": 3, + "value is 33.3333333333, expected 24": 3, + "value is 26.4, expected 24": 3, + "value is 22.4, expected 24": 2, + "value is 63, expected 24": 2, + "value is 105, expected 24": 2, + "value is 4.36363636364, expected 24": 2, + "value is 51.0769230769, expected 24": 2, + "value is 10.8, expected 24": 2, + "value is 6.4, expected 24": 2, + "value is 6.5, expected 24": 2, + "value is 25.6666666667, expected 24": 2, + "value is 50.6666666667, expected 24": 2, + "value is 8.33333333333, expected 24": 2, + "contains disallowed characters": 2, + "value is -36, expected 24": 2, + "value is 17.7777777778, expected 24": 2, + "value is 7.33333333333, expected 24": 2, + "value is 69.3333333333, expected 24": 2, + "value is 132, expected 24": 2, + "value is 14.6666666667, expected 24": 2, + "value is -6, expected 24": 1, + "value is -26, expected 24": 1, + "value is 7.61538461538, expected 24": 1, + "value is 23.1111111111, expected 24": 1, + "value is 117, expected 24": 1, + "value is 24.5, expected 24": 1, + "value is 22.2857142857, expected 24": 1, + "value is 9.42857142857, expected 24": 1, + "value is 78, expected 24": 1, + "value is 51.3333333333, expected 24": 1, + "value is 5.45454545455, expected 24": 1, + "value is 8.4, expected 24": 1, + "value is 7.14285714286, expected 24": 1, + "value is 27.1428571429, expected 24": 1, + "value is 5.6, expected 24": 1, + "value is 98, expected 24": 1, + "value is 23.7777777778, expected 24": 1, + "value is 115.909090909, expected 24": 1, + "value is 29.3333333333, expected 24": 1, + "value is 240, expected 24": 1, + "value is 136, expected 24": 1, + "value is 19.2, expected 24": 1, + "value is 5.71428571429, expected 24": 1, + "value is 22.1428571429, expected 24": 1, + "value is 95, expected 24": 1, + "value is 23.5, expected 24": 1, + "value is 8.125, expected 24": 1, + "value is 160, expected 24": 1, + "value is 23.6, expected 24": 1, + "value is 27.4, expected 24": 1, + "value is 35.2, expected 24": 1, + "value is 53.5714285714, expected 24": 1, + "value is 22.2, expected 24": 1, + "value is 13.5, expected 24": 1, + "value is 28.1666666667, expected 24": 1, + "value is -136, expected 24": 1, + "value is 11.2, expected 24": 1, + "value is 24.6153846154, expected 24": 1, + "value is 21.5, expected 24": 1, + "value is 11, expected 24": 1, + "value is 74.7692307692, expected 24": 1, + "value is 4.91666666667, expected 24": 1, + "value is 47, expected 24": 1, + "value is 65.5, expected 24": 1, + "value is 11.5384615385, expected 24": 1, + "value is 11.25, expected 24": 1, + "value is 21.6363636364, expected 24": 1, + "value is 165, expected 24": 1, + "value is 220, expected 24": 1, + "value is 33.5, expected 24": 1, + "value is 8.66666666667, expected 24": 1, + "value is 3, expected 24": 1, + "value is 23.3333333333, expected 24": 1, + "value is 55, expected 24": 1, + "value is 6.46153846154, expected 24": 1, + "value is 21.3333333333, expected 24": 1, + "value is 168, expected 24": 1, + "value is 32.5714285714, expected 24": 1, + "value is 11.6666666667, expected 24": 1, + "value is 40.4285714286, expected 24": 1, + "value is 93, expected 24": 1, + "value is 24.4, expected 24": 1, + "value is 8.55555555556, expected 24": 1, + "value is 57, expected 24": 1, + "value is -48, expected 24": 1, + "value is 9.6, expected 24": 1, + "value is 18.8571428571, expected 24": 1, + "value is 89.1428571429, expected 24": 1, + "value is 144, expected 24": 1, + "value is 52.25, expected 24": 1, + "value is 16.2, expected 24": 1, + "value is 38.6666666667, expected 24": 1, + "value is -9, expected 24": 1, + "value is 4.66666666667, expected 24": 1, + "value is 126, expected 24": 1, + "value is 12.7272727273, expected 24": 1, + "value is 22.25, expected 24": 1, + "value is 85.3333333333, expected 24": 1, + "value is 4.58333333333, expected 24": 1, + "value is -56, expected 24": 1, + "division by zero": 1, + "value is 33.2, expected 24": 1, + "value is 26.5, expected 24": 1, + "value is -91, expected 24": 1, + "value is 25.3333333333, expected 24": 1, + "value is 75, expected 24": 1 + }, + "empty": false, + "first_attempt_count": 100, + "first_attempt_r1_format_rate": 0.98, + "first_attempt_answer_tag_rate": 0.98, + "first_attempt_allowed_chars_rate": 0.97, + "first_attempt_parse_ok_rate": 0.97, + "first_attempt_numbers_ok_rate": 0.96, + "first_attempt_target_ok_rate": 0.05, + "first_attempt_valid_rate": 0.05, + "attempt_count": 1600, + "attempt_r1_format_rate": 0.98375, + "attempt_answer_tag_rate": 0.98375, + "attempt_allowed_chars_rate": 0.9825, + "attempt_parse_ok_rate": 0.981875, + "attempt_numbers_ok_rate": 0.96375, + "attempt_target_ok_rate": 0.045625, + "attempt_valid_rate": 0.043125, + "mean_candidates_until_solution": 5.888888888888889, + "median_candidates_until_solution": 3.0, + "max_candidates_until_solution": 16, + "solved_at_1_count": 5, + "solved_at_1": 0.05, + "solved_at_4_count": 20, + "solved_at_4": 0.2, + "solved_at_16_count": 36, + "solved_at_16": 0.36, + "num_samples": 16 +} diff --git a/results/union_summary.json b/results/union_summary.json new file mode 100644 index 0000000..0cbef74 --- /dev/null +++ b/results/union_summary.json @@ -0,0 +1,746 @@ +{ + "count": 100, + "solved": 79, + "solved_rate": 0.79, + "total_model_candidates": 25600, + "mean_candidate_budget_per_problem": 256, + "max_candidate_budget_per_problem": 256, + "min_candidate_budget_per_problem": 256, + "mean_candidates_until_solution": 37.51898734177215, + "median_candidates_until_solution": 21, + "max_candidates_until_solution": 164, + "source_budgets": { + "outputs/eval/sample_shaped/shard_0.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_1.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_2.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_3.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_4.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_5.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_6.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_shaped/shard_7.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_0.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_1.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_2.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_3.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_4.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_5.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_6.jsonl": { + "rows": 100, + "candidates": 1600 + }, + "outputs/eval/sample_ch/shard_7.jsonl": { + "rows": 100, + "candidates": 1600 + } + }, + "unsolved_keys": [ + [ + 2, + 3, + 5, + 12 + ], + [ + 3, + 5, + 7, + 13 + ], + [ + 2, + 9, + 13, + 13 + ], + [ + 2, + 4, + 10, + 10 + ], + [ + 1, + 3, + 4, + 6 + ], + [ + 2, + 7, + 7, + 10 + ], + [ + 1, + 4, + 5, + 6 + ], + [ + 7, + 8, + 8, + 13 + ], + [ + 1, + 6, + 6, + 8 + ], + [ + 3, + 3, + 8, + 8 + ], + [ + 6, + 9, + 9, + 10 + ], + [ + 4, + 7, + 11, + 13 + ], + [ + 1, + 6, + 11, + 13 + ], + [ + 2, + 5, + 6, + 9 + ], + [ + 2, + 2, + 6, + 9 + ], + [ + 2, + 2, + 7, + 10 + ], + [ + 4, + 4, + 7, + 13 + ], + [ + 8, + 8, + 9, + 12 + ], + [ + 6, + 6, + 7, + 11 + ], + [ + 8, + 9, + 11, + 11 + ], + [ + 9, + 11, + 12, + 12 + ] + ], + "reason_counts": { + "value is 48, expected 24": 1110, + "ok": 958, + "value is 30, expected 24": 894, + "value is 12, expected 24": 891, + "value is 40, expected 24": 870, + "value is 18, expected 24": 819, + "value is 60, expected 24": 790, + "value is 20, expected 24": 766, + "value is 0, expected 24": 765, + "value is 16, expected 24": 760, + "value is 36, expected 24": 752, + "value is 72, expected 24": 637, + "value is 6, expected 24": 628, + "value is 84, expected 24": 523, + "value is 22, expected 24": 509, + "empty expression": 503, + "numbers are not used exactly once": 495, + "value is 21, expected 24": 487, + "value is 25, expected 24": 471, + "value is 8, expected 24": 471, + "value is 32, expected 24": 452, + "value is 42, expected 24": 440, + "value is 26, expected 24": 399, + "value is 28, expected 24": 394, + "value is 27, expected 24": 360, + "value is 14, expected 24": 343, + "value is 96, expected 24": 339, + "value is 10, expected 24": 308, + "value is 15, expected 24": 298, + "value is 33, expected 24": 293, + "value is 35, expected 24": 260, + "value is 23, expected 24": 246, + "value is 9, expected 24": 226, + "value is 54, expected 24": 218, + "value is 56, expected 24": 217, + "value is 19, expected 24": 216, + "value is 29, expected 24": 216, + "value is 66, expected 24": 166, + "value is 4, expected 24": 161, + "value is 31, expected 24": 147, + "value is 34, expected 24": 145, + "value is 37, expected 24": 140, + "value is 50, expected 24": 132, + "value is 39, expected 24": 122, + "value is 45, expected 24": 120, + "value is 120, expected 24": 120, + "value is -12, expected 24": 117, + "value is 44, expected 24": 112, + "value is 64, expected 24": 106, + "value is 52, expected 24": 105, + "value is 13, expected 24": 103, + "value is 90, expected 24": 102, + "contains disallowed characters": 98, + "value is 70, expected 24": 97, + "value is 9.33333333333, expected 24": 96, + "value is 43, expected 24": 95, + "value is 80, expected 24": 91, + "value is 38, expected 24": 88, + "value is 7, expected 24": 76, + "value is 108, expected 24": 76, + "value is 17, expected 24": 75, + "value is 16.6666666667, expected 24": 75, + "value is 16.5, expected 24": 74, + "value is 10.5, expected 24": 72, + "value is -16, expected 24": 70, + "value is 6.4, expected 24": 63, + "value is 18.6666666667, expected 24": 59, + "value is 3, expected 24": 57, + "value is 77, expected 24": 57, + "value is -6, expected 24": 53, + "value is 5, expected 24": 53, + "value is 78, expected 24": 51, + "value is 20.5714285714, expected 24": 49, + "value is -7, expected 24": 47, + "value is 21.3333333333, expected 24": 46, + "value is 49, expected 24": 45, + "value is 104, expected 24": 45, + "value is 22.5, expected 24": 43, + "value is 12.5, expected 24": 41, + "value is 91, expected 24": 41, + "value is 11, expected 24": 40, + "value is 98, expected 24": 38, + "value is 65, expected 24": 38, + "value is -36, expected 24": 38, + "value is -8, expected 24": 37, + "value is 5.6, expected 24": 34, + "value is 88, expected 24": 34, + "value is -22, expected 24": 34, + "value is 26.4, expected 24": 33, + "value is 63, expected 24": 32, + "value is 33.3333333333, expected 24": 31, + "value is 25.6, expected 24": 27, + "value is 5.71428571429, expected 24": 26, + "value is 4.66666666667, expected 24": 26, + "value is 168, expected 24": 26, + "value is 82.5, expected 24": 26, + "value is 12.8571428571, expected 24": 24, + "value is -136, expected 24": 24, + "value is 17.6, expected 24": 23, + "value is 24.2, expected 24": 23, + "value is 25.6666666667, expected 24": 23, + "value is 32.5714285714, expected 24": 23, + "value is 52.25, expected 24": 21, + "value is -26, expected 24": 20, + "value is 17.3333333333, expected 24": 20, + "value is 6.5, expected 24": 20, + "value is 46, expected 24": 19, + "value is 140, expected 24": 19, + "value is 55, expected 24": 19, + "value is 160, expected 24": 19, + "value is 13.5, expected 24": 19, + "value is 41, expected 24": 18, + "value is 51.0769230769, expected 24": 18, + "value is 6.66666666667, expected 24": 18, + "value is 10.6666666667, expected 24": 18, + "value is 14.4, expected 24": 17, + "value is 9.6, expected 24": 17, + "value is 110, expected 24": 16, + "value is 71, expected 24": 16, + "value is 11.2, expected 24": 16, + "value is 126, expected 24": 16, + "value is 27.1428571429, expected 24": 15, + "value is 70.5, expected 24": 15, + "value is 21.5, expected 24": 14, + "value is 4.5, expected 24": 14, + "value is 13.3333333333, expected 24": 14, + "value is 28.8, expected 24": 14, + "value is 25.6923076923, expected 24": 14, + "value is 25.5, expected 24": 13, + "value is 22.4, expected 24": 13, + "value is 24.6153846154, expected 24": 13, + "value is 29.3333333333, expected 24": 13, + "value is 240, expected 24": 13, + "value is 228, expected 24": 13, + "value is 10.8, expected 24": 13, + "value is 21.6363636364, expected 24": 13, + "value is 15.4285714286, expected 24": 13, + "value is 93, expected 24": 13, + "value is 57, expected 24": 12, + "value is 61, expected 24": 12, + "value is 65.5, expected 24": 12, + "value is 50.6666666667, expected 24": 12, + "value is -20, expected 24": 11, + "value is 132, expected 24": 11, + "division by zero": 11, + "value is 144, expected 24": 11, + "value is 81, expected 24": 11, + "value is 69.3333333333, expected 24": 11, + "value is 47, expected 24": 10, + "value is 24.5, expected 24": 10, + "value is -9, expected 24": 10, + "value is 23.5714285714, expected 24": 10, + "value is 22.8571428571, expected 24": 10, + "value is 68, expected 24": 10, + "value is 58.5, expected 24": 10, + "value is 5.14285714286, expected 24": 10, + "value is 12.6666666667, expected 24": 10, + "value is 8.33333333333, expected 24": 10, + "value is 7.61538461538, expected 24": 9, + "value is 37.3333333333, expected 24": 9, + "value is 4.36363636364, expected 24": 9, + "value is 19.5, expected 24": 9, + "value is 13.7142857143, expected 24": 9, + "value is 8.125, expected 24": 9, + "value is 150, expected 24": 9, + "value is -24, expected 24": 9, + "value is 40.5, expected 24": 9, + "value is 17.8, expected 24": 9, + "value is 5.4, expected 24": 9, + "value is 186, expected 24": 9, + "value is 22.2857142857, expected 24": 8, + "value is 105, expected 24": 8, + "value is 136, expected 24": 8, + "value is 26.5, expected 24": 8, + "value is 8.66666666667, expected 24": 8, + "value is 22.6666666667, expected 24": 8, + "value is 6.54545454545, expected 24": 8, + "value is 8.25, expected 24": 8, + "value is 19.7142857143, expected 24": 8, + "value is 31.6666666667, expected 24": 8, + "value is 35.6, expected 24": 8, + "value is 28.3636363636, expected 24": 8, + "value is 67, expected 24": 8, + "value is 9.5, expected 24": 7, + "value is 117, expected 24": 7, + "value is 8.4, expected 24": 7, + "value is 31.5, expected 24": 7, + "value is 44.5714285714, expected 24": 7, + "value is 2, expected 24": 7, + "value is 18.3333333333, expected 24": 7, + "numbers must be positive integers": 7, + "value is 23.2727272727, expected 24": 7, + "value is 11.6666666667, expected 24": 7, + "value is 89.1428571429, expected 24": 7, + "value is 24.7272727273, expected 24": 7, + "value is 13.6, expected 24": 7, + "value is 23.1111111111, expected 24": 6, + "value is 51.3333333333, expected 24": 6, + "value is 53, expected 24": 6, + "value is 38.5, expected 24": 6, + "value is 23.7777777778, expected 24": 6, + "value is 112, expected 24": 6, + "value is 44.625, expected 24": 6, + "value is 47.2, expected 24": 6, + "value is 21.6, expected 24": 6, + "value is -56, expected 24": 6, + "value is 23.6, expected 24": 6, + "value is -18, expected 24": 6, + "value is 11.5384615385, expected 24": 6, + "value is 17.4, expected 24": 6, + "value is 25.3333333333, expected 24": 6, + "value is 23.25, expected 24": 6, + "value is 17.7777777778, expected 24": 6, + "value is 35.1, expected 24": 6, + "value is 12.7272727273, expected 24": 6, + "value is 8.55555555556, expected 24": 6, + "value is 16.2, expected 24": 6, + "value is 22.25, expected 24": 6, + "value is 75, expected 24": 5, + "value is -15, expected 24": 5, + "value is -65, expected 24": 5, + "value is 9.42857142857, expected 24": 5, + "value is 210, expected 24": 5, + "value is 19.3846153846, expected 24": 5, + "value is -28, expected 24": 5, + "value is 23.3333333333, expected 24": 5, + "value is 9.23076923077, expected 24": 5, + "value is 22.4285714286, expected 24": 5, + "value is 35.2, expected 24": 5, + "value is 20.25, expected 24": 5, + "value is 22.2, expected 24": 5, + "value is 22.6, expected 24": 5, + "value is 21.8333333333, expected 24": 5, + "value is 180, expected 24": 5, + "value is 21.6666666667, expected 24": 5, + "value is 33.6666666667, expected 24": 5, + "value is 40.4285714286, expected 24": 5, + "value is 7.33333333333, expected 24": 5, + "value is 9.81818181818, expected 24": 5, + "value is 38.6666666667, expected 24": 5, + "value is 130, expected 24": 5, + "value is 14.6666666667, expected 24": 5, + "value is 16.3636363636, expected 24": 5, + "value is 28.5, expected 24": 4, + "value is 7.71428571429, expected 24": 4, + "value is 5.77777777778, expected 24": 4, + "value is 51, expected 24": 4, + "value is 11.4285714286, expected 24": 4, + "value is 59, expected 24": 4, + "value is 83, expected 24": 4, + "value is 22.75, expected 24": 4, + "value is 22.1428571429, expected 24": 4, + "value is 19.8, expected 24": 4, + "value is 40.8, expected 24": 4, + "value is 25.0909090909, expected 24": 4, + "value is 27.4, expected 24": 4, + "value is 32.4, expected 24": 4, + "value is 15.25, expected 24": 4, + "value is 25.7142857143, expected 24": 4, + "value is -9.6, expected 24": 4, + "value is 26.6666666667, expected 24": 4, + "value is 243, expected 24": 4, + "value is 39.1111111111, expected 24": 4, + "value is 99, expected 24": 4, + "value is 56.5714285714, expected 24": 4, + "value is 17.4545454545, expected 24": 4, + "value is 5.2, expected 24": 4, + "value is 85.3333333333, expected 24": 4, + "value is -91, expected 24": 4, + "value is 114, expected 24": 3, + "value is 27.5, expected 24": 3, + "value is 37.5, expected 24": 3, + "value is 13.2, expected 24": 3, + "value is 92, expected 24": 3, + "value is 19.6, expected 24": 3, + "value is 7.14285714286, expected 24": 3, + "value is 20.5, expected 24": 3, + "value is 6.85714285714, expected 24": 3, + "value is 7.38461538462, expected 24": 3, + "value is 49.1428571429, expected 24": 3, + "value is 48.7272727273, expected 24": 3, + "value is 72.2857142857, expected 24": 3, + "value is 48.3, expected 24": 3, + "value is 121, expected 24": 3, + "value is 23.5, expected 24": 3, + "value is 22.7692307692, expected 24": 3, + "value is 47.6666666667, expected 24": 3, + "value is 8.61538461538, expected 24": 3, + "value is 22.1666666667, expected 24": 3, + "value is 19.6363636364, expected 24": 3, + "value is 74.7692307692, expected 24": 3, + "value is 68.4, expected 24": 3, + "value is 4.14285714286, expected 24": 3, + "value is 4.91666666667, expected 24": 3, + "value is 76, expected 24": 3, + "value is 45.3333333333, expected 24": 3, + "value is 138, expected 24": 3, + "value is 8.27272727273, expected 24": 3, + "value is 11.25, expected 24": 3, + "value is 19.3333333333, expected 24": 3, + "value is 6.46153846154, expected 24": 3, + "value is 23.1428571429, expected 24": 3, + "value is -48, expected 24": 3, + "value is -2, expected 24": 3, + "value is 36.3636363636, expected 24": 3, + "value is -30, expected 24": 3, + "value is 9.71428571429, expected 24": 3, + "value is 8.16666666667, expected 24": 3, + "value is 7.09090909091, expected 24": 3, + "value is 14.5714285714, expected 24": 3, + "unsupported expression node: UnaryOp": 3, + "value is 36.4, expected 24": 3, + "value is 106, expected 24": 3, + "value is 34.6666666667, expected 24": 2, + "value is 182, expected 24": 2, + "value is 19.2, expected 24": 2, + "value is 5.45454545455, expected 24": 2, + "value is 32.8, expected 24": 2, + "invalid syntax (, line 1)": 2, + "value is -42, expected 24": 2, + "value is 73, expected 24": 2, + "value is 33.4285714286, expected 24": 2, + "value is 33.6, expected 24": 2, + "value is 8.8, expected 24": 2, + "value is 2.75, expected 24": 2, + "value is 17.875, expected 24": 2, + "value is 5.5, expected 24": 2, + "value is 20.4285714286, expected 24": 2, + "value is 22.5714285714, expected 24": 2, + "value is 15.5, expected 24": 2, + "value is 62, expected 24": 2, + "value is 50.5, expected 24": 2, + "value is 29.5, expected 24": 2, + "value is 28.1666666667, expected 24": 2, + "value is 7.27272727273, expected 24": 2, + "value is 18.5, expected 24": 2, + "value is 30.1428571429, expected 24": 2, + "value is 100, expected 24": 2, + "value is 40.8571428571, expected 24": 2, + "value is 15.2, expected 24": 2, + "value is 7.63636363636, expected 24": 2, + "value is 42.3076923077, expected 24": 2, + "value is 1.28571428571, expected 24": 2, + "value is 264, expected 24": 2, + "value is 62.4, expected 24": 2, + "value is 38.4, expected 24": 2, + "value is 7.5, expected 24": 2, + "value is 59.5, expected 24": 2, + "value is 33.5, expected 24": 2, + "value is 24.6666666667, expected 24": 2, + "value is 22.6153846154, expected 24": 2, + "value is 15.1666666667, expected 24": 2, + "value is 15.3333333333, expected 24": 2, + "value is 13.6363636364, expected 24": 2, + "value is 6.92307692308, expected 24": 2, + "value is 4.25, expected 24": 2, + "value is 18.8571428571, expected 24": 2, + "value is 53.8888888889, expected 24": 2, + "value is 41.8181818182, expected 24": 2, + "value is 24.4, expected 24": 2, + "value is 116, expected 24": 2, + "value is 5.90909090909, expected 24": 2, + "value is 7.42857142857, expected 24": 2, + "value is -4.5, expected 24": 2, + "value is 8.75, expected 24": 2, + "value is 26.8571428571, expected 24": 2, + "value is 29.7142857143, expected 24": 2, + "value is 1, expected 24": 2, + "value is 8.44444444444, expected 24": 2, + "value is 66.6666666667, expected 24": 2, + "value is -2.57142857143, expected 24": 2, + "value is 34.5555555556, expected 24": 2, + "value is 15.75, expected 24": 2, + "value is 52.5, expected 24": 2, + "value is -14, expected 24": 2, + "value is 17.5, expected 24": 2, + "value is 24.4615384615, expected 24": 2, + "value is 25.3846153846, expected 24": 2, + "value is 23.4545454545, expected 24": 2, + "value is 19.5555555556, expected 24": 2, + "value is 53.1666666667, expected 24": 2, + "value is 24.75, expected 24": 2, + "value is 33.2, expected 24": 2, + "value is 7.58333333333, expected 24": 2, + "value is 58.1538461538, expected 24": 2, + "value is 14.7777777778, expected 24": 2, + "value is 1164, expected 24": 2, + "value is 14.5, expected 24": 1, + "value is 156, expected 24": 1, + "value is 252, expected 24": 1, + "value is 125, expected 24": 1, + "value is 10.4, expected 24": 1, + "value is 174, expected 24": 1, + "value is 258, expected 24": 1, + "value is 4.85714285714, expected 24": 1, + "value is 10.6363636364, expected 24": 1, + "value is 1.44444444444, expected 24": 1, + "value is 37.7142857143, expected 24": 1, + "value is 85, expected 24": 1, + "value is 440, expected 24": 1, + "value is 23.4285714286, expected 24": 1, + "value is 2.18181818182, expected 24": 1, + "value is 1.85714285714, expected 24": 1, + "value is 4.90909090909, expected 24": 1, + "value is 19.4, expected 24": 1, + "value is 25.2, expected 24": 1, + "value is 11.1428571429, expected 24": 1, + "value is -49, expected 24": 1, + "value is 115.909090909, expected 24": 1, + "value is -2.69230769231, expected 24": 1, + "value is 24.1818181818, expected 24": 1, + "value is -84, expected 24": 1, + "value is 74, expected 24": 1, + "value is 53.3333333333, expected 24": 1, + "value is 62.1428571429, expected 24": 1, + "value is 0.674698795181, expected 24": 1, + "value is 95, expected 24": 1, + "value is 8.72727272727, expected 24": 1, + "value is 242, expected 24": 1, + "value is -3.5, expected 24": 1, + "value is 79.3333333333, expected 24": 1, + "value is 5.625, expected 24": 1, + "value is 184, expected 24": 1, + "value is 4.125, expected 24": 1, + "value is 21.8, expected 24": 1, + "value is -171, expected 24": 1, + "value is 35.8333333333, expected 24": 1, + "value is 51.2, expected 24": 1, + "value is -243, expected 24": 1, + "value is 29.8, expected 24": 1, + "value is 53.5714285714, expected 24": 1, + "value is 8.57142857143, expected 24": 1, + "value is -72, expected 24": 1, + "value is 13.0909090909, expected 24": 1, + "value is 350, expected 24": 1, + "value is 26.8, expected 24": 1, + "value is 17.2, expected 24": 1, + "value is 94.4, expected 24": 1, + "value is 55.25, expected 24": 1, + "value is 2.15384615385, expected 24": 1, + "value is 30.5714285714, expected 24": 1, + "value is 24.7142857143, expected 24": 1, + "value is 17.5384615385, expected 24": 1, + "value is -66, expected 24": 1, + "value is 16.6153846154, expected 24": 1, + "value is 11.0769230769, expected 24": 1, + "value is -5, expected 24": 1, + "value is 27.25, expected 24": 1, + "value is -32, expected 24": 1, + "value is 21.25, expected 24": 1, + "value is 6.25, expected 24": 1, + "value is 9.375, expected 24": 1, + "value is 4.61538461538, expected 24": 1, + "value is 7.69230769231, expected 24": 1, + "value is 43.5, expected 24": 1, + "value is 6.16666666667, expected 24": 1, + "value is 25.8, expected 24": 1, + "value is 18.4285714286, expected 24": 1, + "value is 21.8571428571, expected 24": 1, + "value is 16.2222222222, expected 24": 1, + "value is 24.6, expected 24": 1, + "value is 4.4, expected 24": 1, + "value is 1.45454545455, expected 24": 1, + "value is 165, expected 24": 1, + "value is 220, expected 24": 1, + "value is 300, expected 24": 1, + "value is 19.6666666667, expected 24": 1, + "value is 20.3333333333, expected 24": 1, + "value is 27.3333333333, expected 24": 1, + "value is 9.75, expected 24": 1, + "value is 46.2857142857, expected 24": 1, + "value is -50, expected 24": 1, + "value is 4.44444444444, expected 24": 1, + "value is 94, expected 24": 1, + "value is 48.9, expected 24": 1, + "value is 97.8, expected 24": 1, + "value is 162, expected 24": 1, + "value is 247, expected 24": 1, + "value is 175, expected 24": 1, + "value is 29.6666666667, expected 24": 1, + "value is 217, expected 24": 1, + "value is 64.1666666667, expected 24": 1, + "value is 20.4, expected 24": 1, + "value is -26.6666666667, expected 24": 1, + "value is 36.8181818182, expected 24": 1, + "value is 22.8, expected 24": 1, + "value is 45.1, expected 24": 1, + "value is 24.25, expected 24": 1, + "value is 9.77777777778, expected 24": 1, + "value is 12.8, expected 24": 1, + "value is 23.625, expected 24": 1, + "value is -2.12903225806, expected 24": 1, + "value is 141.166666667, expected 24": 1, + "value is 23.8333333333, expected 24": 1, + "value is 80.5714285714, expected 24": 1, + "value is 54.6666666667, expected 24": 1, + "value is -14.4, expected 24": 1, + "value is 74.6666666667, expected 24": 1, + "value is 29.1428571429, expected 24": 1, + "value is 46.6666666667, expected 24": 1, + "value is 61.9090909091, expected 24": 1, + "value is 4.42857142857, expected 24": 1, + "value is 74.1818181818, expected 24": 1, + "value is 26.75, expected 24": 1, + "value is -3, expected 24": 1, + "value is -52, expected 24": 1, + "value is 17.1428571429, expected 24": 1, + "value is 12.6, expected 24": 1, + "value is -2.55555555556, expected 24": 1, + "value is 14.8571428571, expected 24": 1, + "value is 29.25, expected 24": 1, + "value is 27.75, expected 24": 1, + "value is 6.6, expected 24": 1, + "value is 2.4, expected 24": 1, + "value is -10, expected 24": 1, + "value is 4.58333333333, expected 24": 1, + "value is 0.96, expected 24": 1, + "value is 17.1111111111, expected 24": 1, + "value is 20.2222222222, expected 24": 1, + "value is 19.4444444444, expected 24": 1, + "value is 69, expected 24": 1, + "value is 38.7272727273, expected 24": 1, + "value is 29.1666666667, expected 24": 1, + "value is 5.83333333333, expected 24": 1, + "value is 23.2222222222, expected 24": 1, + "value is 23.375, expected 24": 1, + "value is 62.5, expected 24": 1, + "value is 80.5, expected 24": 1, + "value is 5.38461538462, expected 24": 1, + "value is 32.5, expected 24": 1, + "value is 1.05405405405, expected 24": 1, + "value is 81.5, expected 24": 1, + "value is 46.8, expected 24": 1, + "value is 135, expected 24": 1, + "value is 61.2, expected 24": 1, + "value is 22.9090909091, expected 24": 1, + "value is 24.3333333333, expected 24": 1, + "value is 30.6666666667, expected 24": 1 + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5668a4a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}