coneml-810m-alpha-arithmetic/eval/summary.json

{
  "schema_version": 1,
  "model": "ConeML/coneml-810m-alpha-arithmetic",
  "scope": "Aggregate results from ConeML internal held-out instruments and the stated public-benchmark subsets. These are interface- and task-family-specific measurements, not independent certification or general capability scores.",
  "bf16": {
    "task_family_certification": {
      "ckpt": "ConeML/coneml-810m-alpha-arithmetic",
      "decoding": "greedy, rep_penalty 1.15, frame 'Question:/Answer:'",
      "by_category": {
        "add-1d": {
          "correct": 71,
          "n": 71,
          "acc": 1.0
        },
        "add-2d-carry": {
          "correct": 484,
          "n": 500,
          "acc": 0.968
        },
        "comparison": {
          "correct": 395,
          "n": 500,
          "acc": 0.79
        },
        "missing-addend": {
          "correct": 337,
          "n": 500,
          "acc": 0.674
        },
        "missing-factor": {
          "correct": 90,
          "n": 500,
          "acc": 0.18
        },
        "mul-1d": {
          "correct": 45,
          "n": 45,
          "acc": 1.0
        },
        "mul-2dx1d": {
          "correct": 418,
          "n": 500,
          "acc": 0.836
        },
        "sub-borrow": {
          "correct": 493,
          "n": 500,
          "acc": 0.986
        },
        "two-step": {
          "correct": 219,
          "n": 500,
          "acc": 0.438
        },
        "which-bigger": {
          "correct": 281,
          "n": 300,
          "acc": 0.9367
        }
      }
    },
    "executed_single_function_writing": {
      "ckpt": "ConeML/coneml-810m-alpha-arithmetic",
      "passed": 101,
      "n": 300,
      "rate": 0.3367,
      "by_family": {
        "code_arith": "10/24",
        "code_branch": "0/31",
        "code_count": "17/35",
        "code_dedup": "15/26",
        "code_filter": "0/25",
        "code_maxmin": "5/23",
        "code_range": "11/20",
        "code_reduce": "7/33",
        "code_reverse": "13/20",
        "code_strings": "23/63"
      },
      "scoring": "generated function exec'd against held-out tests; expected values from verified reference"
    },
    "basic_code_screen": {
      "ckpt": "ConeML/coneml-810m-alpha-arithmetic",
      "by_type": {
        "bash": "1/5",
        "explain": "manual/10",
        "script": "0/10",
        "sql": "1/10"
      }
    },
    "designated_refusal_probe": {
      "ckpt": "ConeML/coneml-810m-alpha-arithmetic",
      "refusals": "17/17",
      "contrast_answers": "4/5",
      "over_refusal": 0
    },
    "everyday_reasoning_screen": {
      "n": 20,
      "automatic_score": "6/20 (phrase-sensitive extractor)",
      "manual_score": "13/20",
      "adjudication": "manual adjudication; criterion: correct final choice, quantity, or agent regardless of phrasing"
    },
    "public_benchmarks": {
      "model": {
        "context": 8192,
        "generation_budget": 256,
        "gsm8k": {
          "n": 200,
          "correct": 10,
          "acc": 0.05,
          "truncation_rate": 0.02
        },
        "humaneval": {
          "n": 164,
          "passed": 3,
          "pass@1": 0.0183,
          "truncation_rate": 0.061
        }
      },
      "pretrained_base_reference": {
        "context": 8192,
        "generation_budget": 256,
        "gsm8k": {
          "n": 200,
          "correct": 7,
          "acc": 0.035,
          "truncation_rate": 0.315
        },
        "humaneval": {
          "n": 164,
          "passed": 0,
          "pass@1": 0.0,
          "truncation_rate": 0.665
        }
      }
    },
    "raw_completion_spot_check": {
      "n": 12,
      "seed": 188,
      "temperature": 0.8,
      "repetition_penalty": 1.15,
      "repeated_4gram_rate": 0.0,
      "distinct_2gram_rate": 0.9793,
      "mean_words": 46.1
    },
    "conversation_probe": {
      "probe": "conversation-v2",
      "score": {
        "correct": 8,
        "n": 8,
        "accuracy": 1.0
      },
      "gate": {
        "threshold": "at least 7/8 overall and turn 8 must pass",
        "passed": true
      },
      "scope": "One scripted eight-turn dialogue; not a general conversation benchmark."
    }
  },
  "row_level_evidence": {
    "public_subset": "representative-samples.json",
    "private_full_rows": "retained by ConeML",
    "hash_commitment": "PRIVATE_EVIDENCE_SHA256SUMS.txt"
  }
}