初始化项目，由ModelHub XC社区提供模型

Model: rpant/iolai26-solve
Source: Original Platform

solver/align.py

@@ -0,0 +1,135 @@
+"""Word alignment from tiny parallel corpora, pure python.
+
+Two complementary signals:
+1. Minimal-pair set difference: if two sentence pairs differ in exactly one
+   token on each side, those tokens correspond. Exact and high-precision;
+   these puzzles are constructed to contain such pairs.
+2. Dice co-occurrence over the whole pair set: soft alignment for everything
+   the minimal pairs don't cover.
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from itertools import combinations
+from typing import Dict, List, Tuple
+
+from .preprocess import Pair, strip_punct, tokenize
+
+
+def _toks(s: str) -> List[str]:
+    return [strip_punct(t).casefold() for t in tokenize(s) if strip_punct(t)]
+
+
+def minimal_pair_links(pairs: List[Pair]) -> Counter:
+    """Set-difference alignment: for every pair of examples whose source sides
+    differ by exactly one token multiset element and likewise on target, link
+    the differing tokens. Returns Counter[(src_tok, tgt_tok)] link strengths."""
+    links: Counter = Counter()
+    toks = [(Counter(_toks(p.src)), Counter(_toks(p.tgt))) for p in pairs]
+    for (s1, t1), (s2, t2) in combinations(toks, 2):
+        ds1, ds2 = s1 - s2, s2 - s1
+        dt1, dt2 = t1 - t2, t2 - t1
+        # exactly one differing token on each side, in both examples
+        if sum(ds1.values()) == 1 and sum(ds2.values()) == 1 \
+                and sum(dt1.values()) == 1 and sum(dt2.values()) == 1:
+            a1, a2 = next(iter(ds1)), next(iter(ds2))
+            b1, b2 = next(iter(dt1)), next(iter(dt2))
+            links[(a1, b1)] += 2  # strong: attested by contrast
+            links[(a2, b2)] += 2
+        # shared residue: tokens present in both examples also co-align weakly
+    return links
+
+
+def dice_scores(pairs: List[Pair]) -> Dict[Tuple[str, str], float]:
+    """Dice coefficient between source and target tokens across examples."""
+    src_count: Counter = Counter()
+    tgt_count: Counter = Counter()
+    co: Counter = Counter()
+    for p in pairs:
+        st, tt = set(_toks(p.src)), set(_toks(p.tgt))
+        for a in st:
+            src_count[a] += 1
+        for b in tt:
+            tgt_count[b] += 1
+        for a in st:
+            for b in tt:
+                co[(a, b)] += 1
+    return {
+        (a, b): 2 * c / (src_count[a] + tgt_count[b])
+        for (a, b), c in co.items()
+    }
+
+
+def _morph_backoff(pairs: List[Pair], scores) -> None:
+    """Substring evidence from single-word glosses: if (moko = dog) is
+    attested and token `namoko` co-occurs with `dog`, boost (namoko, dog) —
+    inflected forms inherit their stem's translation. Applied in place."""
+    word_pairs = [
+        (_toks(p.src)[0], _toks(p.tgt)[0])
+        for p in pairs
+        if len(_toks(p.src)) == 1 and len(_toks(p.tgt)) == 1
+    ]
+    for (a, b) in list(scores.keys()):
+        for w, x in word_pairs:
+            if x == b and len(w) >= 3 and w in a and w != a:
+                scores[(a, b)] += 2.0  # inflected src contains attested stem
+            if w == a and len(x) >= 3 and x in b and x != b:
+                scores[(a, b)] += 2.0  # inflected tgt contains attested stem
+
+
+def align(pairs: List[Pair]) -> Dict[str, List[Tuple[str, float]]]:
+    """Combined alignment: src token -> ranked [(tgt token, score)].
+
+    Minimal-pair links dominate (score offset +1.0 per link unit); Dice fills
+    in the rest; single-word glosses back off into inflected forms containing
+    them. Scores are comparable only within one puzzle.
+    """
+    links = minimal_pair_links(pairs)
+    dice = dice_scores(pairs)
+    scores: Dict[Tuple[str, str], float] = defaultdict(float)
+    for k, v in dice.items():
+        scores[k] += v
+    for k, v in links.items():
+        scores[k] += 1.0 * v
+    _morph_backoff(pairs, scores)
+    # competition ("explaining away"): a target token strongly claimed by
+    # some other source is a worse candidate — demote it proportionally to
+    # its best competing suitor. Breaks the pervasive co-occurrence ties of
+    # 10-sentence corpora in favor of unclaimed targets.
+    best_suitor: Dict[str, float] = defaultdict(float)
+    second_suitor: Dict[str, float] = defaultdict(float)
+    for (a, b), s in scores.items():
+        if s > best_suitor[b]:
+            second_suitor[b] = best_suitor[b]
+            best_suitor[b] = s
+        elif s > second_suitor[b]:
+            second_suitor[b] = s
+    out: Dict[str, List[Tuple[str, float]]] = defaultdict(list)
+    for (a, b), s in scores.items():
+        rival = second_suitor[b] if s >= best_suitor[b] else best_suitor[b]
+        out[a].append((b, s - 0.3 * rival))
+    for a in out:
+        out[a].sort(key=lambda x: -x[1])
+    return dict(out)
+
+
+def one_to_one(pairs: List[Pair]) -> Dict[str, str]:
+    """Greedy 1:1 token alignment: highest-scoring links assigned first, each
+    token used once. Sharper than independent argmax when several tokens tie
+    on co-occurrence (small corpora make ties common)."""
+    amap = align(pairs)
+    edges = [(s, a, b) for a, cands in amap.items() for b, s in cands]
+    edges.sort(key=lambda e: (-e[0], e[1], e[2]))
+    taken_a, taken_b, out = set(), set(), {}
+    for s, a, b in edges:
+        if a not in taken_a and b not in taken_b:
+            out[a] = b
+            taken_a.add(a)
+            taken_b.add(b)
+    return out
+
+
+def best_translation(align_map: Dict[str, List[Tuple[str, float]]], tok: str) -> str:
+    cands = align_map.get(tok.casefold(), [])
+    return cands[0][0] if cands else ""

solver/analogy.py

@@ -0,0 +1,98 @@
+"""Proportional analogy: solve a : b :: c : x at the string level.
+
+Used for (1) generating unseen inflected forms from paradigm neighbors and
+(2) the chrF-floor fallback. Transformation model: a -> b is a prefix and/or
+suffix replacement around a shared core, which covers concatenative
+morphology. The same edit is applied to c. All consistent answers are
+returned, ranked by preserved stem material.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import List, Optional, Tuple
+
+# A rule is (a_pre, b_pre, a_suf, b_suf): replace prefix a_pre with b_pre and
+# suffix a_suf with b_suf.
+Rule = Tuple[str, str, str, str]
+
+
+def _lcp(a: str, b: str) -> int:
+    i = 0
+    while i < min(len(a), len(b)) and a[i] == b[i]:
+        i += 1
+    return i
+
+
+def _lcsuf(a: str, b: str) -> int:
+    i = 0
+    while i < min(len(a), len(b)) and a[-1 - i] == b[-1 - i]:
+        i += 1
+    return i
+
+
+def edit_rules(a: str, b: str) -> List[Rule]:
+    """Candidate decompositions of the transformation a -> b."""
+    rules: List[Rule] = []
+    p = _lcp(a, b)
+    s = _lcsuf(a, b)
+    if p > 0:
+        rules.append(("", "", a[p:], b[p:]))  # keep shared prefix, swap suffix
+    if s > 0:
+        rules.append((a[: len(a) - s], b[: len(b) - s], "", ""))  # swap prefix
+    if p > 0 and s > 0 and p + s <= min(len(a), len(b)):
+        # circumfix-ish: shared prefix AND suffix, swap the middle — model as
+        # suffix swap on the part after the shared prefix
+        rules.append(("", "", a[p : len(a) - s], b[p : len(b) - s]))
+    if not rules:
+        rules.append((a, b, "", ""))  # suppletion: whole-string replacement
+    return rules
+
+
+def apply_rule(rule: Rule, c: str) -> Optional[str]:
+    a_pre, b_pre, a_suf, b_suf = rule
+    out = c
+    if a_pre and not out.startswith(a_pre):
+        return None
+    out = b_pre + out[len(a_pre):]
+    if a_suf:
+        if not out.endswith(a_suf):
+            return None
+        out = out[: len(out) - len(a_suf)] + b_suf
+    else:
+        out = out + b_suf
+    return out
+
+
+def apply_rule_mid(rule: Rule, c: str) -> Optional[str]:
+    """Apply a middle-swap rule (encoded as suffix-swap) as an infix
+    substitution when the plain application fails: replace the last
+    occurrence of a_suf inside c."""
+    _, _, a_mid, b_mid = rule
+    if not a_mid or a_mid not in c:
+        return None
+    i = c.rfind(a_mid)
+    return c[:i] + b_mid + c[i + len(a_mid):]
+
+
+def solve(a: str, b: str, c: str) -> List[str]:
+    """Candidate solutions x to a : b :: c : x, best first."""
+    scored: Counter = Counter()
+    for rule in edit_rules(a, b):
+        x = apply_rule(rule, c)
+        if x is None:
+            x = apply_rule_mid(rule, c)
+        if x:
+            stem_kept = len(c) - len(rule[0]) - len(rule[2])
+            scored[x] = max(scored[x], stem_kept)
+    return [w for w, _ in scored.most_common()]
+
+
+def solve_from_pairs(pairs: List[Tuple[str, str]], c: str) -> List[str]:
+    """Given attested (form_a, form_b) pairs exhibiting one transformation,
+    vote for the best x completing c : x under that transformation."""
+    votes: Counter = Counter()
+    for a, b in pairs:
+        for rank, x in enumerate(solve(a, b, c)):
+            votes[x] += 1.0 / (1 + rank)
+    return [w for w, _ in votes.most_common()]

solver/budget.py

@@ -0,0 +1,40 @@
+"""Adaptive compute allocation under the 30-minute wall clock.
+
+Symbolic solvers cost ~0; the budget really governs LLM calls (CEGIS rounds
+and best-of-N). Strategy: reserve a safety margin, spread the rest over the
+LLM-needing puzzles, and degrade rounds/N as the clock runs down — never let
+the tail of the test set hit the fallback because the head overspent.
+"""
+
+from __future__ import annotations
+
+import time
+
+
+class Budget:
+    def __init__(self, total_seconds: float = 1620.0, safety_margin: float = 120.0):
+        self.start = time.monotonic()
+        self.total = total_seconds
+        self.safety = safety_margin
+
+    def elapsed(self) -> float:
+        return time.monotonic() - self.start
+
+    def remaining(self) -> float:
+        return self.total - self.safety - self.elapsed()
+
+    def exhausted(self) -> bool:
+        return self.remaining() <= 0
+
+    def cegis_rounds(self, puzzles_left: int, seconds_per_round: float = 12.0) -> int:
+        """How many CEGIS refinement rounds this puzzle can afford, assuming
+        every remaining puzzle needs at least one proposal."""
+        if puzzles_left <= 0:
+            puzzles_left = 1
+        per_puzzle = self.remaining() / puzzles_left
+        rounds = int(per_puzzle / seconds_per_round) - 1  # -1 for the initial proposal
+        return max(0, min(rounds, 3))
+
+    def allow_llm(self, puzzles_left: int, seconds_per_call: float = 12.0) -> bool:
+        """False once only fallback-speed work fits for the remaining set."""
+        return self.remaining() > puzzles_left * 0.2 + seconds_per_call

solver/direct.py

@@ -0,0 +1,280 @@
+"""LLM answering: prompts, generation, and output parsing (scaffolded and lean)."""
+
+from __future__ import annotations
+
+import re
+from typing import List, Optional, Sequence, Tuple
+
+from .llm import LLMClient
+from .preprocess import Puzzle
+
+SYSTEM = (
+    "You solve International Linguistics Olympiad problems. Every problem is "
+    "about a language you have never seen; ALL the evidence you need is in the "
+    "given data. Derive the rules ONLY from that data — do not assume the "
+    "language works like English or any language you know. "
+    "A 'Mechanical analysis' section may provide segmentation, alignment, and "
+    "candidate answers computed by exact algorithms: use them as hypotheses, "
+    "adopt candidates that fit the data, and correct them when the data "
+    "disagrees. "
+    "First, reason briefly about the linguistic patterns — keep it to a few "
+    "lines, not an essay: line up the given examples, segment the words into "
+    "morphemes, note what each recurring morpheme means and the order they "
+    "combine in, and any sound changes; check that the pattern holds across the "
+    "examples, then apply it to each query item. "
+    "Answer format by task type -- translation: the translated form only, in "
+    "the language asked for; fill_blanks: only the missing form for each "
+    "blank; match_letters: only the option letter (for example A, B, C); "
+    "text_to_num: the number in digits; num_to_text: the number written out "
+    "in the puzzle language; anything else: exactly what the instruction "
+    "asks, nothing more. Match the punctuation style of the given examples. "
+    "Then write a line that says exactly "
+    "FINAL ANSWERS: and, below it, one answer per line in the order the items "
+    "are asked -- the bare answer only, no numbering, no quotes, no markdown, "
+    "no extra text. If you are unsure, still give your single best guess for "
+    "every item -- never leave an item blank. After the answers, write a line "
+    "that says exactly EXPLANATION: followed by 2-4 short bullet points "
+    "stating the rules you found (word order, affixes and their functions, "
+    "sound changes, numeral bases) and the key evidence for them. Do not "
+    "repeat your reasoning.\n\n"
+    "Output shape (follow it exactly; do not bold or number the markers):\n"
+    "<a few lines of pattern reasoning>\n"
+    "FINAL ANSWERS:\n"
+    "first answer\n"
+    "second answer\n"
+    "EXPLANATION:\n"
+    "- rule and evidence\n"
+    "- rule and evidence"
+)
+
+# Minimal prompt: no scaffold, no chain-of-thought. Selected by LEAN_MODE.
+LEAN_SYSTEM = (
+    "You solve International Linguistics Olympiad problems. Answer every "
+    "numbered item. Put each answer on its own line, in order, with no "
+    "numbering and no extra text. Give your best guess for every item; never "
+    "leave one blank."
+)
+
+MAX_NEW_TOKENS = 1536
+
+# Markers tolerate leading markdown/quote decoration (#, *, >, -, spaces) and
+# trailing decoration/colon, and capture any inline content after the marker
+# ("FINAL ANSWERS: foo" -> "foo" is the first answer). Qwen habitually bolds
+# these headers; the strict "^marker$" form silently dropped every such output.
+# trailing class excludes newline ([^\S\n]) so the marker never swallows the
+# line break and misreads the next line as inline content
+_FINAL_RX = re.compile(r"(?im)^[ \t#>*_`-]*final[ \t]*answers?\b[^\S\n]*[:*_`]?[^\S\n]*(.*)$")
+_EXPL_RX = re.compile(r"(?im)^[ \t#>*_`-]*explanation\b[^\S\n]*[:*_`]?[^\S\n]*(.*)$")
+_NUMBERING_RX = re.compile(r"^\s*\(?\d{1,3}[.)]\s*")
+# leading markdown decoration on an answer line: bullets, bold, backticks
+_ANSWER_DECOR_RX = re.compile(r"^[\s>*_`•·–-]+")
+
+
+def build_prompt(puzzle: Puzzle, scaffold: str = "") -> str:
+    base = f"{puzzle.context.strip()}\n\n{puzzle.query.strip()}"
+    if scaffold:
+        base += f"\n\n{scaffold}"
+    return base
+
+
+def _clean_answer_line(line: str) -> str:
+    line = _NUMBERING_RX.sub("", line.strip())
+    line = _ANSWER_DECOR_RX.sub("", line)
+    # strip trailing markdown emphasis but keep sentence punctuation (EM-significant)
+    line = re.sub(r"[*_`]+$", "", line)
+    return line.strip().strip("'\"“”").strip()
+
+
+def parse_output(text: str, n_items: Optional[int] = None
+                 ) -> Tuple[List[str], str, bool]:
+    """Returns (answers, explanation, found_marker).
+
+    `found_marker` is True only when an explicit FINAL ANSWERS marker was
+    present. On a parse failure (no marker) we return ([], expl_if_any, False)
+    rather than treating the reasoning prose as answers -- the caller then
+    keeps its symbolic answers instead of overwriting them with garbage.
+    When `n_items` is given it is used only as a sanity cap on how many
+    answer lines to accept (guards against a runaway list)."""
+    markers = list(_FINAL_RX.finditer(text))
+    found = bool(markers)
+    if markers:
+        m = markers[-1]
+        # a marker may carry its first answer inline: "FINAL ANSWERS: foo"
+        inline = m.group(1).strip() if m.lastindex else ""
+        tail = (inline + "\n" if inline else "") + text[m.end():]
+    else:
+        tail = ""
+
+    expl = ""
+    em = _EXPL_RX.search(tail) if found else None
+    if em:
+        inline_e = em.group(1).strip() if em.lastindex else ""
+        rest_e = tail[em.end():].strip()
+        expl = (inline_e + ("\n" + rest_e if rest_e else "")) if inline_e else rest_e
+        tail = tail[: em.start()]
+
+    answers: List[str] = []
+    cap = (2 * n_items + 4) if n_items else None
+    for line in tail.splitlines():
+        cleaned = _clean_answer_line(line)
+        if cleaned:
+            answers.append(cleaned)
+            if cap and len(answers) >= cap:
+                break
+
+    # models sometimes put EXPLANATION before FINAL ANSWERS despite the prompt
+    if not expl:
+        em2 = _EXPL_RX.search(text)
+        if em2 and (not markers or em2.start() < markers[-1].start()):
+            seg = text[em2.end():]
+            stop = _FINAL_RX.search(seg)
+            expl = (seg[: stop.start()] if stop else seg).strip()
+    return answers, expl, found
+
+
+# lines the model may prepend/append around a bare answer list
+_CHATTY_RX = re.compile(
+    r"^(?:here (?:are|is)\b|answers?\s*:?\s*$|the answers?\b|translations?\s*:?\s*$|"
+    r"note\b|okay\b|sure\b|solution\b|let me\b)", re.IGNORECASE)
+
+
+def parse_output_lean(text: str, n_items: Optional[int] = None
+                      ) -> Tuple[List[str], str, bool]:
+    """Lean parse: the minimal prompt asks for bare answers, one per line, with
+    no FINAL ANSWERS marker — so every non-empty line IS an answer. We only
+    strip numbering/markdown decoration and drop obvious chatty preamble lines.
+    When more than n_items lines survive, keep the LAST n_items (any stray
+    preamble sits at the top). No marker requirement, no salvage."""
+    if not text:
+        return [], "", False
+    answers: List[str] = []
+    for line in text.splitlines():
+        cleaned = _clean_answer_line(line)
+        if cleaned and not _CHATTY_RX.match(cleaned):
+            answers.append(cleaned)
+    if n_items and len(answers) > n_items:
+        answers = answers[-n_items:]
+    return answers, "", bool(answers)
+
+
+_LEAN_LABEL_RX = re.compile(r"^\s*\(?(\d{1,3})\)?[.):\]]\s")
+
+
+def align_lean(text: str, puzzle: Puzzle) -> List[Optional[str]]:
+    """Turn a lean (bare-lines) model output into exactly len(puzzle.items)
+    answers, blending two placement methods that stack:
+
+    1. LABEL-AWARE placement — if the model numbered its answer lines and those
+       numbers cover most of the item labels, place each answer under its own
+       label. This is robust to the model reordering items or skipping one (a
+       single skipped line would otherwise shift every later answer and zero the
+       whole block on both metrics).
+    2. POSITIONAL last-N fallback — when the output isn't reliably numbered, take
+       the cleaned, non-chatty lines in order (dropping any preamble at the top).
+
+    Both share our line hygiene (numbering/markdown stripping via
+    `_clean_answer_line`, preamble drop via `_CHATTY_RX`). Returns a length-N
+    list; None marks items the model did not answer, which the caller leaves to
+    the fallback."""
+    items = puzzle.items
+    n = len(items)
+    if n == 0:
+        return []
+    labeled: dict = {}
+    for line in (text or "").splitlines():
+        s = line.strip()
+        if not s:
+            continue
+        m = _LEAN_LABEL_RX.match(s)
+        cleaned = _clean_answer_line(s)
+        if not cleaned or _CHATTY_RX.match(cleaned):
+            continue
+        if m:
+            labeled[m.group(1)] = cleaned  # last write wins (models restate)
+    item_labels = [it.number for it in items]
+    if labeled and all(lbl for lbl in item_labels):
+        covered = sum(1 for lbl in item_labels if lbl in labeled)
+        if covered >= max(1, (2 * n + 2) // 3):  # ~2/3 labelled -> trust labels
+            return [labeled.get(lbl) for lbl in item_labels]
+    # positional fallback: our bare-line parse, aligned by position
+    bare, _e, _f = parse_output_lean(text, n)
+    out: List[Optional[str]] = [None] * n
+    for i in range(min(n, len(bare))):
+        out[i] = bare[i]
+    return out
+
+
+def build_salvage_prompt(base_prompt: str, raw_reasoning: str,
+                         max_chars: int = 2400) -> str:
+    """A short follow-up prompt that reuses reasoning the model already
+    produced (but never terminated with a FINAL ANSWERS block, e.g. it hit the
+    token cap). We feed the reasoning back as context and ask ONLY for the
+    answer block -- cheaper and more reliable than a bare retry."""
+    reasoning = raw_reasoning.strip()
+    if len(reasoning) > max_chars:
+        reasoning = reasoning[-max_chars:]  # keep the most recent reasoning
+    return (
+        f"{base_prompt}\n\n"
+        "You already worked through this problem:\n"
+        "-----\n"
+        f"{reasoning}\n"
+        "-----\n"
+        "Now output ONLY the answer block, nothing else. Write the line "
+        "FINAL ANSWERS: then one bare answer per line in item order (your best "
+        "guess for every item, never blank), then a line EXPLANATION: with 2-4 "
+        "short bullets."
+    )
+
+
+def solve_direct(puzzles: Sequence[Puzzle], client: LLMClient,
+                 scaffolds: Optional[Sequence[str]] = None,
+                 max_new_tokens: int = MAX_NEW_TOKENS,
+                 salvage: bool = True,
+                 system: str = SYSTEM,
+                 lean: bool = False,
+                 sample: bool = False,
+                 temperature: float = 0.5
+                 ) -> List[Tuple[List[str], str, str, bool]]:
+    """Batched single-shot answering. Returns per puzzle
+    (answers, explanation, raw_text, found_marker). `sample`/`temperature` drive
+    the sampled passes used by self-consistency voting; salvage is skipped when
+    sampling. In `lean` mode every non-empty line is an answer (no marker)."""
+    if not client.available or not puzzles:
+        return [([], "", "", False) for _ in puzzles]
+    prompts = [
+        build_prompt(p, scaffolds[i] if scaffolds else "")
+        for i, p in enumerate(puzzles)
+    ]
+    raws = client.generate(prompts, max_new_tokens=max_new_tokens, system=system,
+                           sample=sample, temperature=temperature)
+    out: List[Tuple[List[str], str, str, bool]] = []
+    salvage_idx: List[int] = []
+    for i, r in enumerate(raws):
+        if lean:
+            aligned = align_lean(r, puzzles[i])
+            answers, expl, found = aligned, "", any(a for a in aligned)
+        else:
+            answers, expl, found = parse_output(r, len(puzzles[i].items))
+        out.append((answers, expl, r, found))
+        if salvage and not lean and not sample and not found and r.strip():
+            salvage_idx.append(i)
+
+    if salvage_idx:
+        sp = [build_salvage_prompt(prompts[i], raws[i]) for i in salvage_idx]
+        # answers only — a small cap keeps the salvage pass cheap
+        cap = min(max_new_tokens, 384)
+        sraws = client.generate(sp, max_new_tokens=cap, system=system)
+        for i, sr in zip(salvage_idx, sraws):
+            a2, e2, f2 = parse_output(sr, len(puzzles[i].items))
+            if f2 and a2:
+                prev = out[i]
+                out[i] = (a2, e2 or prev[1], prev[2], True)
+    return out
+
+
+def align_answers(direct: List[str], n_items: int) -> List[Optional[str]]:
+    """Position-align a FINAL ANSWERS block to the expected item count."""
+    out: List[Optional[str]] = [None] * n_items
+    for i in range(min(n_items, len(direct))):
+        out[i] = direct[i]
+    return out

solver/dsl/grammar.py

@@ -0,0 +1,188 @@
+"""DSL datatypes for puzzle grammars.
+
+A Grammar is a small, fully-executable description of one puzzle language:
+lexicon + affixes + rewrite rules + constituent order (+ numerals). The LLM
+proposes grammars as JSON; `from_json` parses defensively (a malformed rule
+is dropped, never fatal). `mdl` gives the description length used by the
+verifier's simplicity penalty.
+
+JSON shape the proposer LLM emits:
+{
+  "lexicon":  [{"morph": "kupu", "gloss": "bird", "pos": "N"}, ...],
+  "affixes":  [{"position": "prefix"|"suffix"|"circumfix", "form": "na",
+                "form2": "", "feature": "DEF", "trigger": "N"}, ...],
+  "rewrites": [{"pattern": "a+a", "repl": "aa", "context": ""}, ...]   # regex
+  "order":    ["V", "S", "O"],          # target constituent order
+  "agree":    [{"src_slot": "S", "dst_slot": "V", "feature": "NUM"}, ...],
+  "redup":    [{"scope": "first_syllable", "feature": "PL"}, ...]
+}
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+
+@dataclass
+class LexEntry:
+    morph: str
+    gloss: str
+    pos: str = ""
+    features: Dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class Affix:
+    position: str  # prefix | suffix | infix | circumfix
+    form: str
+    feature: str = ""  # what it marks, e.g. "PL", "PST", "DEF"
+    trigger: str = ""  # pos or feature it attaches to; "" = any
+    form2: str = ""  # second part for circumfix
+
+
+@dataclass
+class Rewrite:
+    pattern: str  # regex over the surface string
+    repl: str
+    context: str = ""  # optional regex that must match for rule to fire
+
+    def apply(self, s: str) -> str:
+        try:
+            if self.context and not re.search(self.context, s):
+                return s
+            return re.sub(self.pattern, self.repl, s)
+        except re.error:
+            return s
+
+
+@dataclass
+class Agree:
+    src_slot: str
+    dst_slot: str
+    feature: str
+
+
+@dataclass
+class Redup:
+    scope: str = "first_syllable"  # or "full", "first_cv"
+    feature: str = "PL"
+
+
+@dataclass
+class Grammar:
+    lexicon: List[LexEntry] = field(default_factory=list)
+    affixes: List[Affix] = field(default_factory=list)
+    rewrites: List[Rewrite] = field(default_factory=list)
+    order: List[str] = field(default_factory=list)
+    agree: List[Agree] = field(default_factory=list)
+    redup: List[Redup] = field(default_factory=list)
+    notes: str = ""
+
+    # ---- lookup helpers ----
+    def by_gloss(self) -> Dict[str, LexEntry]:
+        return {e.gloss.casefold(): e for e in self.lexicon}
+
+    def by_morph(self) -> Dict[str, LexEntry]:
+        return {e.morph: e for e in self.lexicon}
+
+    def mdl(self) -> float:
+        """Description length: total symbols in the grammar. Lightly weighted
+        by the verifier; only breaks ties between equally-fitting grammars."""
+        n = 0
+        for e in self.lexicon:
+            n += len(e.morph) + len(e.gloss) + 2
+        for a in self.affixes:
+            n += len(a.form) + len(a.form2) + len(a.feature) + 3
+        for r in self.rewrites:
+            n += len(r.pattern) + len(r.repl) + len(r.context) + 3
+        n += 2 * len(self.order) + 4 * len(self.agree) + 4 * len(self.redup)
+        return float(n)
+
+    def to_json(self) -> str:
+        return json.dumps(
+            {
+                "lexicon": [
+                    {"morph": e.morph, "gloss": e.gloss, "pos": e.pos, "features": e.features}
+                    for e in self.lexicon
+                ],
+                "affixes": [
+                    {"position": a.position, "form": a.form, "form2": a.form2,
+                     "feature": a.feature, "trigger": a.trigger}
+                    for a in self.affixes
+                ],
+                "rewrites": [
+                    {"pattern": r.pattern, "repl": r.repl, "context": r.context}
+                    for r in self.rewrites
+                ],
+                "order": self.order,
+                "agree": [
+                    {"src_slot": g.src_slot, "dst_slot": g.dst_slot, "feature": g.feature}
+                    for g in self.agree
+                ],
+                "redup": [{"scope": d.scope, "feature": d.feature} for d in self.redup],
+            },
+            ensure_ascii=False,
+        )
+
+
+def _get(d: dict, *keys: str, default: str = "") -> str:
+    for k in keys:
+        if k in d and d[k] is not None:
+            return str(d[k])
+    return default
+
+
+def from_json(text: str) -> Optional[Grammar]:
+    """Parse an LLM-emitted grammar. Tolerates surrounding prose/code fences
+    and drops malformed entries instead of failing."""
+    m = re.search(r"\{.*\}", text, re.DOTALL)
+    if not m:
+        return None
+    try:
+        data = json.loads(m.group(0))
+    except json.JSONDecodeError:
+        # common LLM damage: trailing commas
+        try:
+            data = json.loads(re.sub(r",\s*([}\]])", r"\1", m.group(0)))
+        except json.JSONDecodeError:
+            return None
+    if not isinstance(data, dict):
+        return None
+
+    g = Grammar()
+    for e in data.get("lexicon") or []:
+        if isinstance(e, dict):
+            morph, gloss = _get(e, "morph", "form", "word"), _get(e, "gloss", "meaning")
+            if morph and gloss:
+                feats = e.get("features") if isinstance(e.get("features"), dict) else {}
+                g.lexicon.append(LexEntry(morph, gloss, _get(e, "pos"), {str(k): str(v) for k, v in (feats or {}).items()}))
+    for a in data.get("affixes") or []:
+        if isinstance(a, dict):
+            form = _get(a, "form")
+            pos = _get(a, "position", default="suffix").lower()
+            if form and pos in ("prefix", "suffix", "infix", "circumfix"):
+                g.affixes.append(Affix(pos, form, _get(a, "feature", "gloss"), _get(a, "trigger"), _get(a, "form2")))
+    for r in data.get("rewrites") or []:
+        if isinstance(r, dict) and _get(r, "pattern"):
+            try:
+                re.compile(_get(r, "pattern"))
+                if _get(r, "context"):
+                    re.compile(_get(r, "context"))
+            except re.error:
+                continue
+            g.rewrites.append(Rewrite(_get(r, "pattern"), _get(r, "repl", "replacement"), _get(r, "context")))
+    order = data.get("order") or []
+    if isinstance(order, list):
+        g.order = [str(x) for x in order]
+    for ag in data.get("agree") or []:
+        if isinstance(ag, dict) and _get(ag, "feature"):
+            g.agree.append(Agree(_get(ag, "src_slot", "src"), _get(ag, "dst_slot", "dst"), _get(ag, "feature")))
+    for rd in data.get("redup") or []:
+        if isinstance(rd, dict):
+            g.redup.append(Redup(_get(rd, "scope", default="first_syllable"), _get(rd, "feature", default="PL")))
+    if not g.lexicon and not g.affixes and not g.rewrites:
+        return None
+    return g

solver/dsl/interpreter.py

@@ -0,0 +1,198 @@
+"""Deterministic interpreter for DSL grammars: generate() (work -> task
+language) and analyze() (task -> work language).
+
+The interpreter is intentionally strict and simple: it executes exactly what
+the grammar says. If a grammar needs cleverness, the proposer must encode it
+(e.g. list 'birds' as its own lexicon entry instead of relying on affix
+machinery). The verifier then selects grammars that this interpreter executes
+into correct outputs — that closed loop is the whole design.
+
+Conventions the proposer prompt establishes:
+- lexicon glosses are work-language words/phrases (may be multiword);
+- affix `feature` is the work-language cue it realizes: a function word
+  ("the", "will", "not") or a marker name ("plural") — during generation a
+  feature fires when its cue appears in the work sentence next to the stem;
+- `order` is a list of pos tags giving target-language constituent order;
+- rewrites are surface regex applied after morph concatenation (word-level).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional, Tuple
+
+from .grammar import Affix, Grammar, LexEntry
+
+
+def _words(s: str) -> List[str]:
+    return [w for w in re.findall(r"[^\s]+", s.strip()) if w]
+
+
+def _clean(w: str) -> str:
+    return w.strip(",;.!?()[]\"'«»").casefold()
+
+
+class Interpreter:
+    def __init__(self, grammar: Grammar):
+        self.g = grammar
+        # gloss index: multiword glosses first (longest match wins)
+        self._gloss_entries: List[Tuple[List[str], LexEntry]] = sorted(
+            (( [_clean(w) for w in _words(e.gloss)], e) for e in grammar.lexicon if e.gloss),
+            key=lambda t: -len(t[0]),
+        )
+        self._morphs: Dict[str, LexEntry] = {e.morph: e for e in grammar.lexicon}
+        self._affix_by_cue: Dict[str, List[Affix]] = {}
+        for a in grammar.affixes:
+            self._affix_by_cue.setdefault(_clean(a.feature), []).append(a)
+
+    # ------------------------------------------------------------------
+    # generation: work-language sentence -> task-language string
+    # ------------------------------------------------------------------
+    def generate(self, work_sentence: str) -> Optional[str]:
+        toks = [_clean(w) for w in _words(work_sentence)]
+        if not toks:
+            return None
+        n = len(toks)
+        used = [False] * n
+        stems: List[Tuple[int, LexEntry]] = []  # (position of first gloss word, entry)
+
+        # 1. cover with lexicon glosses, longest first
+        for gloss_words, entry in self._gloss_entries:
+            L = len(gloss_words)
+            i = 0
+            while i + L <= n:
+                if not any(used[i : i + L]) and toks[i : i + L] == gloss_words:
+                    for j in range(i, i + L):
+                        used[j] = True
+                    stems.append((i, entry))
+                    i += L
+                else:
+                    i += 1
+        if not stems:
+            return None
+        stems.sort(key=lambda t: t[0])
+
+        # 2. leftover tokens fire affixes on the nearest eligible stem
+        pending: Dict[int, List[Affix]] = {k: [] for k in range(len(stems))}
+        uncovered = [i for i in range(n) if not used[i]]
+        for i in uncovered:
+            cue = toks[i]
+            for a in self._affix_by_cue.get(cue, []):
+                k = self._nearest_stem(stems, i, a)
+                if k is not None:
+                    pending[k].append(a)
+                    break
+
+        # 3. order stems by target constituent order if pos info available
+        idx = list(range(len(stems)))
+        if self.g.order and all(e.pos for _, e in stems):
+            rank = {pos: r for r, pos in enumerate(self.g.order)}
+            idx.sort(key=lambda k: (rank.get(stems[k][1].pos, len(rank)), stems[k][0]))
+
+        # 4. build surface words: affix attachment then rewrites
+        out_words = []
+        for k in idx:
+            _, entry = stems[k]
+            w = entry.morph
+            for a in pending[k]:
+                w = self._attach(w, a)
+            w = self._apply_rewrites(w)
+            out_words.append(w)
+        surface = " ".join(out_words)
+        return self._apply_rewrites_sentence(surface)
+
+    def _nearest_stem(self, stems: List[Tuple[int, LexEntry]], cue_pos: int, affix: Affix) -> Optional[int]:
+        best_k, best_d = None, 10 ** 9
+        for k, (pos, entry) in enumerate(stems):
+            if affix.trigger and affix.trigger not in (entry.pos, entry.gloss):
+                continue
+            d = abs(pos - cue_pos)
+            if d < best_d:
+                best_k, best_d = k, d
+        return best_k
+
+    def _attach(self, w: str, a: Affix) -> str:
+        if a.position == "prefix":
+            return a.form + w
+        if a.position == "suffix":
+            return w + a.form
+        if a.position == "circumfix":
+            return a.form + w + (a.form2 or a.form)
+        if a.position == "infix":
+            # after the first vowel-less onset (common infix site: after first C)
+            m = re.match(r"^([^aeiouAEIOU]*)(.*)$", w)
+            return (m.group(1) + a.form + m.group(2)) if m else w + a.form
+        return w
+
+    def _apply_rewrites(self, w: str) -> str:
+        for r in self.g.rewrites:
+            w = r.apply(w)
+        return w
+
+    def _apply_rewrites_sentence(self, s: str) -> str:
+        # rewrites with explicit spaces/anchors act at sentence level too
+        for r in self.g.rewrites:
+            if " " in r.pattern or r.pattern.startswith("^") or r.pattern.endswith("$"):
+                s = r.apply(s)
+        return s
+
+    # ------------------------------------------------------------------
+    # analysis: task-language sentence -> work-language string
+    # ------------------------------------------------------------------
+    def analyze(self, task_sentence: str) -> Optional[str]:
+        words = [_clean(w) for w in _words(task_sentence)]
+        if not words:
+            return None
+        rendered: List[Tuple[str, str, str]] = []  # (prefix cues, gloss, suffix cues)
+        any_known = False
+        for w in words:
+            pre, gloss, post, known = self._analyze_word(w)
+            any_known = any_known or known
+            rendered.append((pre, gloss, post))
+        if not any_known:
+            return None
+        out: List[str] = []
+        for pre, gloss, post in rendered:
+            for c in pre.split():
+                out.append(c)
+            out.append(gloss)
+            for c in post.split():
+                out.append(c)
+        return " ".join(x for x in out if x)
+
+    def _analyze_word(self, w: str) -> Tuple[str, str, str, bool]:
+        """Decompose one surface word into (prefix cues, stem gloss, suffix
+        cues, matched?). Tries direct lexicon hit, then affix stripping
+        (longest affix first), then returns the word untouched."""
+        if w in self._morphs:
+            return "", self._morphs[w].gloss, "", True
+        affixes = sorted(self.g.affixes, key=lambda a: -len(a.form))
+        for a in affixes:
+            if a.position == "prefix" and w.startswith(a.form):
+                pre, gloss, post, ok = self._analyze_word(w[len(a.form):])
+                if ok:
+                    return (self._cue(a) + " " + pre).strip(), gloss, post, True
+            if a.position == "suffix" and w.endswith(a.form):
+                pre, gloss, post, ok = self._analyze_word(w[: len(w) - len(a.form)])
+                if ok:
+                    return pre, gloss, (post + " " + self._cue(a)).strip(), True
+            if a.position == "circumfix" and w.startswith(a.form) and w.endswith(a.form2 or a.form):
+                inner = w[len(a.form): len(w) - len(a.form2 or a.form)]
+                pre, gloss, post, ok = self._analyze_word(inner)
+                if ok:
+                    return (self._cue(a) + " " + pre).strip(), gloss, post, True
+        # last resort: greedy stem containment (rewrite rules may have altered edges)
+        for morph, entry in sorted(self._morphs.items(), key=lambda kv: -len(kv[0])):
+            if len(morph) >= 3 and morph in w:
+                return "", entry.gloss, "", True
+        return "", w, "", False
+
+    @staticmethod
+    def _cue(a: Affix) -> str:
+        """How an affix surfaces in the work-language output: function-word
+        cues are emitted verbatim; abstract markers (PL, PST) are dropped —
+        the proposer should prefer word cues for translatable material."""
+        cue = a.feature.strip()
+        if cue and cue.isalpha() and cue.casefold() == cue:
+            return cue
+        return ""

solver/fallback.py

@@ -0,0 +1,74 @@
+"""chrF-floor fallback: never return an empty or wildly-off answer.
+
+The geometric-mean metric means one empty answer costs far more than a wrong
+but plausible one. Fallback ladder (best available wins):
+1. analogy from the closest attested source (transfers its target with the
+   observed source->query edit applied),
+2. the attested target of the most chrF-similar attested source,
+3. echo of query content words mapped through alignment,
+4. the raw query text itself (last resort: shares characters with gold more
+   often than an empty string does).
+"""
+
+from __future__ import annotations
+
+from typing import List, Optional, Tuple
+
+from . import analogy
+from .metrics import chrf
+from .align import align as build_align, best_translation
+from .preprocess import Pair, strip_punct, tokenize
+
+
+def closest_attested(query: str, sources: List[str]) -> Tuple[int, float]:
+    """Index and similarity of the attested source closest to the query."""
+    best_i, best_s = -1, -1.0
+    for i, s in enumerate(sources):
+        sc = chrf(query, s)
+        if sc > best_s:
+            best_i, best_s = i, sc
+    return best_i, best_s
+
+
+def fallback_answer(query: str, pairs: List[Pair], direction: str = "to_work") -> str:
+    """direction: 'to_work' = translate task->work (analysis);
+    'to_task' = work->task (generation). Pairs are (task, work)."""
+    if direction == "to_task":
+        srcs = [p.tgt for p in pairs]
+        tgts = [p.src for p in pairs]
+        flipped = [Pair(src=p.tgt, tgt=p.src) for p in pairs]
+    else:
+        srcs = [p.src for p in pairs]
+        tgts = [p.tgt for p in pairs]
+        flipped = pairs
+
+    query = query.strip()
+    if not query:
+        return tgts[0] if tgts else "?"
+
+    if srcs:
+        i, sim = closest_attested(query, srcs)
+        if i >= 0:
+            # 1. analogy transfer: apply the srcs[i]->query edit to tgts[i]
+            transfer = analogy.solve(srcs[i], query, tgts[i])
+            if transfer and sim > 0.3:
+                return transfer[0]
+            # 2. echo the closest attested target
+            if sim > 0.15 and tgts[i]:
+                return tgts[i]
+
+    # 3. word-by-word through alignment
+    amap = build_align(flipped)
+    words = [strip_punct(t) for t in tokenize(query)]
+    mapped = [best_translation(amap, w) or w for w in words if w]
+    if mapped:
+        return " ".join(mapped)
+
+    # 4. absolute floor
+    return query
+
+
+def ensure_nonempty(ans: Optional[str], query: str, pairs: List[Pair], direction: str = "to_work") -> str:
+    if ans and str(ans).strip():
+        return str(ans).strip()
+    return fallback_answer(query, pairs, direction) or "?"

solver/llm.py

@@ -0,0 +1,346 @@
+"""LLM clients: HFTransformersClient (transformers/AWQ, T4, greedy) plus
+NullClient/CallableClient for dev and an optional VLLMClient."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Callable, List, Optional, Sequence, Tuple
+
+MAX_NEW_TOKENS = 1024
+DEFAULT_MODEL_DIR = "."  # submission: weights ship at the repo root
+
+
+class LLMClient:
+    """Interface: generate(prompts) -> one completion per prompt."""
+
+    available: bool = False
+    # can this backend score candidate next-tokens (needed for the match_letters
+    # assignment solver)? Only the real transformers backend can.
+    can_score: bool = False
+    deadline: Optional[float] = None  # monotonic wall-clock abort (armed by caller)
+
+    def generate(self, prompts: Sequence[str], max_new_tokens: int = MAX_NEW_TOKENS,
+                 system: Optional[str] = None, sample: bool = False,
+                 temperature: float = 0.5) -> List[str]:
+        raise NotImplementedError
+
+
+class NullClient(LLMClient):
+    available = False
+
+    def generate(self, prompts, max_new_tokens=MAX_NEW_TOKENS, system=None,
+                 sample=False, temperature=0.5):
+        return ["" for _ in prompts]
+
+
+class CallableClient(LLMClient):
+    available = True
+
+    def __init__(self, fn: Callable[[str], str]):
+        self.fn = fn
+
+    def generate(self, prompts, max_new_tokens=MAX_NEW_TOKENS, system=None,
+                 sample=False, temperature=0.5):
+        return [self.fn(p) for p in prompts]
+
+
+def pack_by_tokens(lengths: Sequence[int], token_budget: int, max_batch: int
+                   ) -> List[List[int]]:
+    """Group prompt indices into batches whose TOTAL token count stays under
+    `token_budget`. On the sandbox's transformers version, prefill computes
+    float32 logits over every prompt position (batch x seq x 152k vocab), so
+    total-tokens-per-batch — not batch count — is what bounds T4 memory.
+    An oversized single prompt still gets its own batch (handled by the OOM
+    retry path)."""
+    batches: List[List[int]] = []
+    cur: List[int] = []
+    cur_tokens = 0
+    for i, n in enumerate(lengths):
+        if cur and (cur_tokens + n > token_budget or len(cur) >= max_batch):
+            batches.append(cur)
+            cur, cur_tokens = [], 0
+        cur.append(i)
+        cur_tokens += n
+    if cur:
+        batches.append(cur)
+    return batches
+
+
+class HFTransformersClient(LLMClient):
+    """transformers backend, mirroring the official notebook's loading code
+    (fp16, device_map=auto, greedy) plus token-budget batching and OOM
+    recovery. Import is lazy."""
+
+    available = True
+    can_score = True  # supports score_next_logprobs (match_letters assignment)
+
+    # total prompt tokens per generation batch. On the sandbox's transformers,
+    # prefill computes fp32 logits over every prompt position (batch x seq x
+    # 152k vocab), ~0.9 MB/token; total-tokens-per-batch — not batch count — is
+    # the T4 memory bound. 3500 is the safe fallback used when we cannot
+    # measure free VRAM; __init__ raises it to fit whatever headroom the loaded
+    # model actually leaves (≈6500 for a 7B-AWQ, ≈3500 for a 14B-AWQ).
+    TOKEN_BUDGET = 3500
+    MB_PER_TOKEN = 0.9      # fp32 prefill logits at Qwen's 152k vocab
+    VRAM_RESERVE_GB = 1.8   # KV cache + activations + fragmentation slack
+
+    def __init__(self, model_dir: str = DEFAULT_MODEL_DIR, batch_size: int = 8):
+        import inspect
+        import time as _time
+
+        import torch
+        from transformers import (AutoModelForCausalLM, AutoTokenizer,
+                                  StoppingCriteria, StoppingCriteriaList)
+
+        self.torch = torch
+        self.tok = AutoTokenizer.from_pretrained(model_dir)
+        # Pin to GPU 0: device_map="auto" can silently offload layers to CPU on a
+        # tight T4 (~100x slower). Fall back to "auto" if the pinned load fails.
+        try:
+            self.model = AutoModelForCausalLM.from_pretrained(
+                model_dir, torch_dtype=torch.float16,
+                device_map={"": 0} if torch.cuda.is_available() else "auto",
+            ).eval()
+        except Exception:
+            self.model = AutoModelForCausalLM.from_pretrained(
+                model_dir, torch_dtype=torch.float16, device_map="auto",
+            ).eval()
+        if self.tok.pad_token_id is None:
+            self.tok.pad_token = self.tok.eos_token
+        self.batch_size = batch_size
+        # newer transformers can skip full-sequence prefill logits entirely
+        fwd_params = inspect.signature(self.model.forward).parameters
+        self._logits_kwarg = next(
+            (k for k in ("logits_to_keep", "num_logits_to_keep") if k in fwd_params),
+            None)
+        self._tune_token_budget()
+        self.last_truncated: List[bool] = []
+
+        # per-token wall-clock abort: a batch started near the deadline can't
+        # overrun and get the process killed (the budget is otherwise only
+        # checked between batches). set self.deadline (monotonic ts) to arm it.
+        self.deadline: Optional[float] = None
+
+        class _Deadline(StoppingCriteria):
+            def __call__(self, input_ids, scores, **kw):
+                return _time.monotonic() > deadline_holder[0]
+
+        deadline_holder = [float("inf")]
+        self._deadline_holder = deadline_holder
+        self._deadline_crit = StoppingCriteriaList([_Deadline()])
+
+    def _tune_token_budget(self) -> None:
+        """Size the per-batch token budget to the VRAM the loaded weights
+        actually leave free. Falls back to the safe class default if the GPU
+        can't be queried (CPU dev, older CUDA)."""
+        torch = self.torch
+        try:
+            if not torch.cuda.is_available():
+                return
+            free_bytes, _ = torch.cuda.mem_get_info()
+            free_gb = free_bytes / 1e9
+            budget = int((free_gb - self.VRAM_RESERVE_GB) * 1000 / self.MB_PER_TOKEN)
+            # the logits_to_keep fast path removes the big allocation entirely,
+            # but stay conservative regardless; clamp to a sane window
+            self.TOKEN_BUDGET = max(2500, min(7000, budget))
+        except Exception:
+            pass
+
+    def _chat_texts(self, prompts: Sequence[str], system: Optional[str]) -> List[str]:
+        texts = []
+        for p in prompts:
+            messages = ([{"role": "system", "content": system}] if system else []) \
+                + [{"role": "user", "content": p}]
+            texts.append(self.tok.apply_chat_template(
+                messages, add_generation_prompt=True, tokenize=False))
+        return texts
+
+    def _generate_chunk(self, chunk: List[str], max_new_tokens: int,
+                        sample: bool = False, temperature: float = 0.5
+                        ) -> Tuple[List[str], List[bool]]:
+        torch = self.torch
+        # pad only when batching more than one prompt (padding can perturb a
+        # quantized model's greedy outputs)
+        enc = self.tok(chunk, return_tensors="pt", padding=len(chunk) > 1,
+                       add_special_tokens=False).to(self.model.device)
+        kwargs = {self._logits_kwarg: 1} if self._logits_kwarg else {}
+        # repetition_penalty=1.0 explicitly: the shipped generation_config sets
+        # 1.05, which is applied even under greedy and biases against the
+        # repeated characters common in these answers.
+        kwargs["repetition_penalty"] = 1.0
+        if sample:
+            kwargs.update(do_sample=True, temperature=temperature, top_p=0.95)
+        else:
+            kwargs["do_sample"] = False
+        if self.deadline is not None:
+            self._deadline_holder[0] = self.deadline
+            kwargs["stopping_criteria"] = self._deadline_crit
+        with torch.no_grad():
+            gen = self.model.generate(
+                **enc, max_new_tokens=max_new_tokens,
+                pad_token_id=self.tok.pad_token_id, **kwargs,
+            )
+        new_tokens = gen[:, enc["input_ids"].shape[1]:]
+        eos = self.tok.eos_token_id
+        texts, truncated = [], []
+        for row in new_tokens:
+            texts.append(_guard(self.tok.decode(row, skip_special_tokens=True)).strip())
+            # no EOS in the generated span => generation was cut at the cap
+            truncated.append(eos is None or int((row == eos).sum()) == 0)
+        return texts, truncated
+
+    def generate(self, prompts, max_new_tokens=MAX_NEW_TOKENS, system=None,
+                 sample=False, temperature=0.5):
+        if not prompts:
+            self.last_truncated = []
+            return []
+        torch = self.torch
+        texts = self._chat_texts(prompts, system)
+        lengths = [len(self.tok(t, add_special_tokens=False)["input_ids"])
+                   for t in texts]
+        out: List[str] = [""] * len(texts)
+        trunc: List[bool] = [False] * len(texts)
+        prev_side = self.tok.padding_side
+        self.tok.padding_side = "left"  # sequences must end at the gen position
+        try:
+            for batch in pack_by_tokens(lengths, self.TOKEN_BUDGET, self.batch_size):
+                chunk = [texts[i] for i in batch]
+                try:
+                    results, tflags = self._generate_chunk(
+                        chunk, max_new_tokens, sample, temperature)
+                except torch.cuda.OutOfMemoryError:
+                    # halve pressure: clear cache, retry one prompt at a time;
+                    # a prompt that OOMs alone yields "" (symbolic answer stands)
+                    torch.cuda.empty_cache()
+                    results, tflags = [], []
+                    for t in chunk:
+                        try:
+                            r1, f1 = self._generate_chunk(
+                                [t], max_new_tokens, sample, temperature)
+                            results.append(r1[0])
+                            tflags.append(f1[0])
+                        except torch.cuda.OutOfMemoryError:
+                            torch.cuda.empty_cache()
+                            results.append("")
+                            tflags.append(False)
+                for i, r, f in zip(batch, results, tflags):
+                    out[i] = r
+                    trunc[i] = f
+        finally:
+            self.tok.padding_side = prev_side
+        self.last_truncated = trunc
+        return out
+
+    def score_next_logprobs(self, prompts, cand_token_ids, system=None,
+                            batch_size=4):
+        """For each prompt, return the max next-token log-prob over each
+        candidate group. `cand_token_ids` is a list of token-id groups (one per
+        option), shared across prompts. Returns List[List[float]]
+        (prompt x option). One forward pass per batch; used by the match_letters
+        assignment solver. Missing/OOM/timed-out prompts get all-zero rows so
+        the caller can fall back."""
+        import time
+        if not prompts:
+            return []
+        torch = self.torch
+        n_opt = len(cand_token_ids)
+        texts = self._chat_texts(prompts, system)
+        out: List[List[float]] = []
+        prev_side = self.tok.padding_side
+        self.tok.padding_side = "left"
+        bs = batch_size
+        i = 0
+        try:
+            while i < len(texts):
+                if self.deadline is not None and time.monotonic() > self.deadline:
+                    out.extend([[0.0] * n_opt for _ in range(len(texts) - i)])
+                    break
+                chunk = texts[i:i + bs]
+                try:
+                    enc = self.tok(chunk, return_tensors="pt", padding=True,
+                                   add_special_tokens=False, truncation=True,
+                                   max_length=6144).to(self.model.device)
+                    fwd = {self._logits_kwarg: 1} if self._logits_kwarg else {}
+                    with torch.no_grad():
+                        logits = self.model(**enc, **fwd).logits[:, -1, :].float()
+                    logprobs = torch.log_softmax(logits, dim=-1)
+                    for b in range(len(chunk)):
+                        row = [max((logprobs[b, t].item() for t in group),
+                                   default=-1e9) if group else -1e9
+                               for group in cand_token_ids]
+                        out.append(row)
+                    i += bs
+                except torch.cuda.OutOfMemoryError:
+                    torch.cuda.empty_cache()
+                    if bs == 1:
+                        out.append([0.0] * n_opt)
+                        i += 1
+                    else:
+                        bs = max(1, bs // 2)
+        finally:
+            self.tok.padding_side = prev_side
+        return out
+
+
+class VLLMClient(LLMClient):
+    """Optional vLLM backend for throughput experiments. Never required."""
+
+    available = True
+
+    def __init__(self, model_dir: str = DEFAULT_MODEL_DIR, max_model_len: int = 4096):
+        from vllm import LLM
+
+        self.llm = LLM(model=model_dir, dtype="half", max_model_len=max_model_len,
+                       gpu_memory_utilization=0.90)
+
+    def generate(self, prompts, max_new_tokens=MAX_NEW_TOKENS, system=None):
+        from vllm import SamplingParams
+
+        tok = self.llm.get_tokenizer()
+        texts = []
+        for p in prompts:
+            messages = ([{"role": "system", "content": system}] if system else []) \
+                + [{"role": "user", "content": p}]
+            texts.append(tok.apply_chat_template(messages, add_generation_prompt=True,
+                                                 tokenize=False))
+        params = SamplingParams(temperature=0.0, max_tokens=max_new_tokens)
+        outs = self.llm.generate(texts, params)
+        return [_guard(o.outputs[0].text if o.outputs else "").strip() for o in outs]
+
+
+def _guard(text: str, max_repeat: int = 4) -> str:
+    """Loop-collapse guard: truncate at the point where a line repeats more
+    than `max_repeat` times consecutively."""
+    lines = text.splitlines()
+    out, streak = [], 0
+    for i, l in enumerate(lines):
+        if i > 0 and l.strip() and l == lines[i - 1]:
+            streak += 1
+            if streak >= max_repeat:
+                break
+        else:
+            streak = 0
+        out.append(l)
+    return "\n".join(out)
+
+
+def load_client(model_id: Optional[str] = None) -> LLMClient:
+    """Best available client for `model_id` (script.py's MODEL_ID):
+    - a local path ("." in the submission, weights/base in dev) is loaded
+      when its config.json exists;
+    - a Hub name (contains "/" and is not a local dir) is passed straight to
+      transformers — the Colab-testing path, mirroring the workshop notebook;
+    - anything unloadable degrades to NullClient (symbolic-only pipeline)."""
+    if model_id and "/" in model_id and not Path(model_id).exists():
+        try:
+            return HFTransformersClient(model_dir=model_id)
+        except Exception:
+            return NullClient()
+    for d in ([model_id] if model_id else []) + [DEFAULT_MODEL_DIR, "weights/base"]:
+        if d and Path(d, "config.json").exists():
+            try:
+                return HFTransformersClient(model_dir=d)
+            except Exception:
+                continue
+    return NullClient()

solver/matching.py

@@ -0,0 +1,266 @@
+"""match_letters: optimal one-to-one assignment (pure-python Hungarian) over
+either surface-similarity scores or the model's next-token log-probs."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from .align import align as build_align
+from .preprocess import Pair, Puzzle, strip_punct, tokenize
+
+
+def hungarian(cost: List[List[float]]) -> List[int]:
+    """Minimum-cost perfect matching on a square cost matrix.
+    Returns assignment: row i -> column result[i]. Jonker-style O(n^3)
+    shortest augmenting path implementation."""
+    n = len(cost)
+    if n == 0:
+        return []
+    INF = float("inf")
+    u = [0.0] * (n + 1)
+    v = [0.0] * (n + 1)
+    p = [0] * (n + 1)  # p[j] = row matched to column j (1-indexed)
+    way = [0] * (n + 1)
+    for i in range(1, n + 1):
+        p[0] = i
+        j0 = 0
+        minv = [INF] * (n + 1)
+        used = [False] * (n + 1)
+        while True:
+            used[j0] = True
+            i0, delta, j1 = p[j0], INF, 0
+            for j in range(1, n + 1):
+                if not used[j]:
+                    cur = cost[i0 - 1][j - 1] - u[i0] - v[j]
+                    if cur < minv[j]:
+                        minv[j] = cur
+                        way[j] = j0
+                    if minv[j] < delta:
+                        delta = minv[j]
+                        j1 = j
+            for j in range(n + 1):
+                if used[j]:
+                    u[p[j]] += delta
+                    v[j] -= delta
+                else:
+                    minv[j] -= delta
+            j0 = j1
+            if p[j0] == 0:
+                break
+        while j0:
+            j1 = way[j0]
+            p[j0] = p[j1]
+            j0 = j1
+    ans = [0] * n
+    for j in range(1, n + 1):
+        if p[j]:
+            ans[p[j] - 1] = j - 1
+    return ans
+
+
+def _char_ngrams(s: str, nmin: int = 2, nmax: int = 4) -> set:
+    s = s.casefold().replace(" ", "")
+    return {s[i : i + n] for n in range(nmin, nmax + 1) for i in range(len(s) - n + 1)}
+
+
+def _sim(a: str, b: str) -> float:
+    ga, gb = _char_ngrams(a), _char_ngrams(b)
+    if not ga or not gb:
+        return 0.0
+    return len(ga & gb) / max(len(ga | gb), 1)
+
+
+def score_matrix(
+    forms: Sequence[str], meanings: Sequence[str], pairs: List[Pair]
+) -> List[List[float]]:
+    """Higher = better match. Combines token-level alignment evidence from the
+    attested pairs with surface similarity to attested forms sharing meaning
+    words."""
+    amap = build_align(pairs) if pairs else {}
+    # index attested: meaning word -> attested source strings
+    attested_by_word: Dict[str, List[str]] = {}
+    for p in pairs:
+        for w in tokenize(p.tgt):
+            w = strip_punct(w).casefold()
+            if w:
+                attested_by_word.setdefault(w, []).append(p.src)
+
+    S = []
+    for f in forms:
+        f_toks = [strip_punct(t).casefold() for t in tokenize(f)]
+        row = []
+        for m in meanings:
+            m_words = [strip_punct(w).casefold() for w in tokenize(m)]
+            score = 0.0
+            # alignment evidence: form tokens aligned to meaning words
+            for ft in f_toks:
+                for tgt, s in amap.get(ft, []):
+                    if tgt in m_words:
+                        score += s
+            # surface similarity to attested sources of these meaning words
+            for w in m_words:
+                for src in attested_by_word.get(w, []):
+                    score += 0.5 * _sim(f, src)
+            row.append(score)
+        S.append(row)
+    return S
+
+
+def _self_sim(texts: Sequence[str], char_level: bool) -> List[List[float]]:
+    """Pairwise similarity within one side: shared char n-grams for unknown
+    forms, shared content words for meanings."""
+    n = len(texts)
+    feats = []
+    for t in texts:
+        if char_level:
+            feats.append(_char_ngrams(t, 3, 5))
+        else:
+            stop = {"the", "a", "an", "of", "is", "it", "he", "she", "they",
+                    "are", "in", "to", "for", "with", "and", "or"}
+            feats.append({w for w in (strip_punct(x).casefold() for x in tokenize(t))
+                          if w and w not in stop})
+    S = [[0.0] * n for _ in range(n)]
+    for i in range(n):
+        for j in range(i + 1, n):
+            inter = len(feats[i] & feats[j])
+            union = len(feats[i] | feats[j]) or 1
+            S[i][j] = S[j][i] = inter / union
+    return S
+
+
+def structural_scores(forms: Sequence[str], meanings: Sequence[str],
+                      iters: int = 4) -> List[List[float]]:
+    """Structure-matching signal for zero-lexical-evidence matching: forms
+    sharing morphemes should map to meanings sharing words. Soft assignment
+    power iteration X <- Sf @ X @ Sm (a light quadratic-assignment relaxation)
+    starting from uniform. Returns an (n_forms x n_meanings) score matrix."""
+    nf, nm = len(forms), len(meanings)
+    if nf == 0 or nm == 0:
+        return [[0.0] * nm for _ in range(nf)]
+    Sf = _self_sim(forms, char_level=True)
+    Sm = _self_sim(meanings, char_level=False)
+    # seed with degree-profile agreement: a form clustered with k others
+    # should map to a meaning clustered with ~k others. (A uniform seed is a
+    # degenerate fixed point — every row converges to the same profile.)
+    def profile(S, i):
+        return sorted((v for v in S[i] if v > 0.05), reverse=True)[:6]
+
+    X = []
+    for i in range(nf):
+        pf = profile(Sf, i)
+        row = []
+        for j in range(nm):
+            pm = profile(Sm, j)
+            d = sum(abs(a - b) for a, b in zip(pf, pm)) + abs(len(pf) - len(pm))
+            row.append(1.0 / (1.0 + d))
+        X.append(row)
+    for _ in range(iters):
+        # Y = Sf @ X @ Sm  (tiny n: pure-python is fine)
+        T = [[sum(Sf[i][k] * X[k][j] for k in range(nf)) for j in range(nm)]
+             for i in range(nf)]
+        Y = [[sum(T[i][k] * Sm[k][j] for k in range(nm)) for j in range(nm)]
+             for i in range(nf)]
+        # row-normalize to keep the iteration bounded
+        X = []
+        for row in Y:
+            z = sum(row) or 1.0
+            X.append([v / z for v in row])
+    return X
+
+
+def solve_matching(
+    forms: Sequence[str], meanings: Sequence[str], pairs: List[Pair]
+) -> List[Tuple[str, str]]:
+    """Optimal assignment of forms to meanings. Combines lexical/alignment
+    evidence (when attested pairs exist) with the structural signal (always).
+    Pads to square with zero scores when lengths differ."""
+    n = max(len(forms), len(meanings))
+    S = score_matrix(forms, meanings, pairs)
+    S2 = structural_scores(forms, meanings)
+    cost = [[0.0] * n for _ in range(n)]
+    for i in range(n):
+        for j in range(n):
+            s = 0.0
+            if i < len(forms) and j < len(meanings):
+                s = S[i][j] + 3.0 * len(meanings) * S2[i][j]
+            cost[i][j] = -s
+    assign = hungarian(cost)
+    out = []
+    for i, f in enumerate(forms):
+        j = assign[i]
+        out.append((f, meanings[j] if j < len(meanings) else ""))
+    return out
+
+
+# --------------------------------------------------------------------------
+# LLM-scored assignment. Free-form generation tends to answer match_letters with
+# the identity permutation (A, B, C, ...) — a valid permutation that scores ~0.
+# Instead we score each (item, option-letter) pair from the model's next-token
+# log-probs and take the optimal one-to-one assignment, so the bijection is
+# enforced exactly rather than hoped for. The assignment engine is our existing
+# Hungarian; only the score source changes (surface features -> the model's own
+# distribution).
+# --------------------------------------------------------------------------
+
+_MATCH_SYSTEM = (
+    "You match items to their correct counterparts in a linguistics problem. "
+    "Reply with one option letter only."
+)
+
+
+def letters_from_scores(scores: List[List[float]], letters: Sequence[str]
+                        ) -> List[str]:
+    """Given a per-item score over option letters (item x option), return one
+    letter per item. When items and options are equinumerous (the IOL matching
+    shape) the assignment is a strict bijection via Hungarian; otherwise it
+    degrades to an independent per-item argmax."""
+    n_items, n_opt = len(scores), len(letters)
+    if n_items == 0 or n_opt == 0:
+        return []
+    if n_items == n_opt:
+        cost = [[-scores[i][j] for j in range(n_opt)] for i in range(n_items)]
+        assign = hungarian(cost)
+        return [letters[assign[i]] for i in range(n_items)]
+    return [letters[max(range(n_opt), key=lambda j: scores[i][j])]
+            for i in range(n_items)]
+
+
+def solve_match_letters_llm(puzzle: Puzzle, client) -> Optional[List[str]]:
+    """Assign each numbered item to an option letter using the model's
+    next-token log-probs, then Hungarian. Returns one letter per puzzle item
+    (in item order), or None if it declines: no scoring backend, not a
+    well-formed lettered-matching shape, or fewer than 3 items/options."""
+    if not getattr(client, "can_score", False):
+        return None
+    items = puzzle.items
+    if not puzzle.lettered or len(items) < 3:
+        return None
+    letters = sorted(puzzle.lettered)
+    if len(letters) < 3:
+        return None
+
+    tok = client.tok
+    cand_ids: List[List[int]] = []
+    for L in letters:
+        ids = set()
+        for form in (L, " " + L):
+            t = tok.encode(form, add_special_tokens=False)
+            if t:
+                ids.add(t[0])
+        cand_ids.append(sorted(ids))
+    if any(not g for g in cand_ids):
+        return None
+
+    ctx = (puzzle.context or "").strip()
+    prompts = []
+    for it in items:
+        num = it.number or "?"
+        form = (it.text or "").strip()
+        prompts.append(
+            f"{ctx}\n\nWhich lettered option corresponds to item {num} "
+            f"({form})? Reply with the option letter only.")
+
+    scores = client.score_next_logprobs(prompts, cand_ids, system=_MATCH_SYSTEM)
+    if not scores or len(scores) != len(items):
+        return None
+    return letters_from_scores(scores, letters)

solver/metrics.py

@@ -0,0 +1,132 @@
+"""Metrics matching the official eval notebook (iolai-2026-workshop).
+
+Lives in solver/ (not eval/) because the RUNTIME needs it: the verifier and
+the fallback ladder score candidate answers with chrF/EM at inference time.
+The dev-only eval/ package re-exports from here.
+
+Semantics:
+
+- EM: case-insensitive exact string equality after .strip() ONLY — punctuation
+  and internal whitespace are significant. Gold items may carry alternatives
+  (list of accepted strings); a hit on any alternative counts.
+- chrF: sacrebleu CHRF() defaults (char n-grams 1..6, beta=2, whitespace not
+  in n-grams, epsilon-smoothed per order), 0..1 here (notebook prints 0..100).
+  Max over alternatives.
+- Aggregate: per-item average of each metric; geometric mean reported as the
+  headline (the competition combines EM and chrF; the notebook prints both).
+
+chrF is implemented in pure python replicating sacrebleu's algorithm so the
+eval sandbox needs no dependency; tests/test_scorer.py checks parity against
+sacrebleu when it is installed.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+import unicodedata
+from collections import Counter
+from typing import Dict, List, Optional, Sequence, Union
+
+CHRF_NGRAM_ORDER = 6
+CHRF_BETA = 2.0
+_EPS = 1e-16
+
+Gold = Union[str, Sequence[str]]  # a gold item: one string or alternatives
+
+
+def _alts(gold: Gold) -> List[str]:
+    if isinstance(gold, str):
+        return [gold]
+    return [str(a) for a in gold] or [""]
+
+
+def normalize_answer(s: str) -> str:
+    """Official EM normalization: strip + lowercase. Nothing else — final
+    punctuation and internal spacing are significant."""
+    return str(s).strip().lower()
+
+
+def exact_match(pred: str, gold: Gold) -> float:
+    p = normalize_answer(pred)
+    return 1.0 if any(p == normalize_answer(a) for a in _alts(gold)) else 0.0
+
+
+def _char_ngrams(s: str, n: int) -> Counter:
+    return Counter(s[i : i + n] for i in range(len(s) - n + 1))
+
+
+def _chrf_single(pred: str, gold: str, n_order: int = CHRF_NGRAM_ORDER,
+                 beta: float = CHRF_BETA) -> float:
+    """Exact replication of sacrebleu CHRF defaults (whitespace stripped,
+    effective-order smoothing): precision/recall averaged over orders where
+    BOTH sides have n-grams; hypothesis counts are zeroed for orders the
+    reference lacks. Returns [0, 1] (sacrebleu reports x100)."""
+    pred_s = "".join(str(pred).split())
+    gold_s = "".join(str(gold).split())
+    avg_prec = avg_rec = 0.0
+    effective = 0
+    for n in range(1, n_order + 1):
+        gn = _char_ngrams(gold_s, n)
+        pn = _char_ngrams(pred_s, n)
+        n_ref = sum(gn.values())
+        n_hyp = sum(pn.values()) if gn else 0  # sacrebleu: no ref => no hyp hits
+        if n_hyp > 0 and n_ref > 0:
+            overlap = sum((pn & gn).values())
+            avg_prec += overlap / n_hyp
+            avg_rec += overlap / n_ref
+            effective += 1
+    if effective == 0:
+        return 0.0
+    avg_prec /= effective
+    avg_rec /= effective
+    if avg_prec + avg_rec == 0:
+        return 0.0
+    b2 = beta * beta
+    return (1 + b2) * avg_prec * avg_rec / (b2 * avg_prec + avg_rec)
+
+
+def chrf(pred: str, gold: Gold) -> float:
+    return max(_chrf_single(pred, a) for a in _alts(gold))
+
+
+def item_scores(pred: str, gold: Gold) -> Dict[str, float]:
+    return {"em": exact_match(pred, gold), "chrf": chrf(pred, gold)}
+
+
+def score_submission(
+    preds: Sequence[Sequence[str]],
+    golds: Sequence[Sequence[Gold]],
+    weights: Optional[Sequence[Sequence[float]]] = None,
+) -> Dict[str, float]:
+    """Score a full submission.
+
+    preds/golds: per row, a list of items; each gold item is a string or a
+    list of accepted alternatives. weights: optional per-item point values;
+    uniform if None. Length mismatches within a row are penalized: missing
+    preds score 0, extra preds are ignored (notebook lines preds up by
+    position exactly the same way).
+    """
+    total_w = 0.0
+    em_w = 0.0
+    chrf_w = 0.0
+    n_items = 0
+    for ri, (prow, grow) in enumerate(zip(preds, golds)):
+        wrow = list(weights[ri]) if weights is not None else [1.0] * len(grow)
+        for ii, gold in enumerate(grow):
+            w = wrow[ii] if ii < len(wrow) else 1.0
+            pred = prow[ii] if ii < len(prow) else ""
+            total_w += w
+            em_w += w * exact_match(pred, gold)
+            chrf_w += w * chrf(pred, gold)
+            n_items += 1
+    if total_w == 0:
+        return {"em": 0.0, "chrf": 0.0, "score": 0.0, "n_items": 0}
+    em_avg = em_w / total_w
+    chrf_avg = chrf_w / total_w
+    return {
+        "em": em_avg,
+        "chrf": chrf_avg,
+        "score": math.sqrt(em_avg * chrf_avg),
+        "n_items": n_items,
+    }

solver/numerals.py

@@ -0,0 +1,247 @@
+"""Numeral system induction: recover morpheme values + combination structure
+from attested (numeral phrase, integer) pairs, then convert both directions.
+
+Model (covers the large majority of IOL numeral systems):
+  value(phrase) = fold over tokens, where adjacent (multiplier, base-power)
+  groups combine multiplicatively and groups combine additively — i.e. the
+  standard "mixed-radix polynomial" reading:  [2] [20] [3]  ->  2*20 + 3.
+Some systems are subtractive or overcounting; a signed-additive variant is
+also searched. Token values are solved by constraint search: each distinct
+token gets an unknown integer value; attested equations constrain them.
+
+Search is tiny: numeral puzzles use ~5-15 morpheme types with values drawn
+from {1..9, base, base^2, ...}. We enumerate candidate value sets per token
+from divisors/residues of the attested numbers, then DFS with propagation.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from .preprocess import strip_punct, tokenize
+
+BASES = (10, 20, 5, 12, 60, 4, 6, 8, 15)
+MAX_TOKEN_VALUE = 10_000
+
+
+def _norm_tokens(phrase: str) -> List[str]:
+    toks = []
+    for t in tokenize(phrase.casefold()):
+        t = strip_punct(t)
+        # split on hyphens: numeral compounds are often hyphenated
+        toks.extend([p for p in re.split(r"[-‑]", t) if p])
+    return toks
+
+
+def _eval(vals: Sequence[int]) -> int:
+    """Evaluate token values with the multiplicative-additive convention: a
+    smaller value directly before a larger one multiplies it; otherwise
+    values add. E.g. [2,20,3] -> 2*20+3 = 43; [3,100,20,7] -> 327."""
+    total = 0
+    cur = vals[0]
+    for prev, v in zip(vals, vals[1:]):
+        if v > prev:
+            cur = cur * v  # e.g. 2 then 20 -> 40
+        else:
+            total += cur
+            cur = v
+    return total + cur
+
+
+class NumeralSystem:
+    def __init__(self, values: Dict[str, int]):
+        self.values = dict(values)
+
+    def text_to_num(self, phrase: str) -> Optional[int]:
+        toks = _norm_tokens(phrase)
+        if not toks or any(t not in self.values for t in toks):
+            return None
+        return _eval([self.values[t] for t in toks])
+
+    def num_to_text(self, n: int, attested_phrases: Sequence[str]) -> Optional[str]:
+        """Generate the phrase for n: enumerate token sequences (up to length
+        6) whose evaluation equals n, then pick the one most consistent with
+        the attested phrasing style (e.g. do multi-token phrases always give
+        a base its explicit multiplier, even 'one'?)."""
+        toks = sorted(self.values, key=lambda t: -self.values[t])
+        found: List[List[str]] = []
+        self._search(n, toks, [], 6, found, limit=16, budget=[100_000])
+        if not found:
+            return None
+        style = _StyleModel(self.values, attested_phrases)
+        found.sort(key=lambda seq: (-style.score(seq), len(seq)))
+        return " ".join(found[0])
+
+    def _search(self, target: int, toks: List[str], acc: List[str], depth: int,
+                found: List[List[str]], limit: int, budget: List[int]) -> None:
+        if len(found) >= limit or budget[0] <= 0:
+            return
+        budget[0] -= 1
+        if target == 0 and acc:
+            found.append(list(acc))
+            return
+        if depth == 0 or target <= 0:
+            return
+        for t in toks:
+            v = self.values[t]
+            if v > target:
+                continue
+            # multiplicative: k * v <= target with k attested as token
+            for m in toks:
+                mv = self.values[m]
+                if 1 <= mv < v and mv * v <= target:
+                    self._search(target - mv * v, toks, acc + [m, t], depth - 2,
+                                 found, limit, budget)
+            self._search(target - v, toks, acc + [t], depth - 1, found, limit, budget)
+
+
+class _StyleModel:
+    """Scores a candidate numeral phrase by consistency with attested style:
+    (a) are base tokens (value >= 10) given an explicit smaller multiplier in
+    attested multi-token phrases? (b) reuse of attested token bigrams."""
+
+    def __init__(self, values: Dict[str, int], phrases: Sequence[str]):
+        self.values = values
+        self.bigrams = set()
+        obs: List[bool] = []
+        for ph in phrases:
+            toks = _norm_tokens(ph)
+            if not toks or any(t not in values for t in toks):
+                continue
+            self.bigrams.update(zip(toks, toks[1:]))
+            if len(toks) < 2:
+                continue
+            for i, t in enumerate(toks):
+                if values[t] >= 10:
+                    obs.append(i > 0 and values[toks[i - 1]] < values[t])
+        self.prefer_explicit = sum(obs) > len(obs) / 2 if obs else False
+
+    def score(self, seq: Sequence[str]) -> float:
+        s = 0.0
+        s += 0.5 * sum(1 for bg in zip(seq, seq[1:]) if bg in self.bigrams)
+        base_seen: Counter = Counter()
+        if len(seq) >= 2:
+            for i, t in enumerate(seq):
+                if self.values[t] >= 10:
+                    base_seen[t] += 1
+                    explicit = i > 0 and self.values[seq[i - 1]] < self.values[t]
+                    s += 1.0 if explicit == self.prefer_explicit else -1.0
+        # positional systems use each base power once; repeats are degenerate
+        s -= 2.0 * sum(c - 1 for c in base_seen.values())
+        return s
+
+
+def induce(attested: List[Tuple[str, int]], max_candidates: int = 8) -> Optional[NumeralSystem]:
+    """Induce token values from attested (phrase, value) pairs by DFS with
+    forward checking. Candidate values per token come from structural
+    positions: divisors of attested values, small digits, and base powers."""
+    eqs: List[Tuple[List[str], int]] = []
+    vocab: List[str] = []
+    for phrase, val in attested:
+        toks = _norm_tokens(phrase)
+        if not toks:
+            continue
+        eqs.append((toks, val))
+        for t in toks:
+            if t not in vocab:
+                vocab.append(t)
+    if not eqs:
+        return None
+
+    # Candidate values per token.
+    digits = set(range(1, 10))
+    base_powers = {b ** k for b in BASES for k in (1, 2, 3) if b ** k <= MAX_TOKEN_VALUE}
+    cands: Dict[str, List[int]] = {}
+    for t in vocab:
+        cs = set(digits) | base_powers
+        # a token appearing alone in an equation must equal that value
+        for toks, val in eqs:
+            if toks == [t]:
+                cs = {val}
+                break
+            if t in toks:
+                cs |= {val} | {d for d in _divisors(val) if d <= MAX_TOKEN_VALUE}
+        cands[t] = sorted(cs)
+
+    # Constraint propagation on short equations before search: a 1-token
+    # equation pins its token; a 2-token equation with one token pinned
+    # constrains the other to {V-a, V/a}.
+    changed = True
+    while changed:
+        changed = False
+        for toks, val in eqs:
+            unknown = [t for t in set(toks) if len(cands[t]) > 1]
+            if len(set(toks)) == 1:
+                t = toks[0]
+                if len(toks) == 1 and cands[t] != [val]:
+                    cands[t] = [val]
+                    changed = True
+            elif len(toks) == 2 and len(unknown) == 1:
+                t = unknown[0]
+                other = toks[0] if toks[1] == t else toks[1]
+                if len(cands[other]) == 1:
+                    a = cands[other][0]
+                    allowed = {val - a}
+                    if a and val % a == 0:
+                        allowed.add(val // a)
+                    new = [v for v in cands[t] if v in allowed]
+                    if new and new != cands[t]:
+                        cands[t] = new
+                        changed = True
+
+    # Order: most-constrained tokens first.
+    order = sorted(vocab, key=lambda t: len(cands[t]))
+
+    assignment: Dict[str, int] = {}
+    budget = {"nodes": 200_000}
+
+    def consistent() -> bool:
+        for toks, val in eqs:
+            if all(t in assignment for t in toks):
+                if _eval([assignment[t] for t in toks]) != val:
+                    return False
+        return True
+
+    def dfs(i: int) -> bool:
+        if budget["nodes"] <= 0:
+            return False  # search space too big — abstain, don't hang
+        if i == len(order):
+            return True
+        t = order[i]
+        for v in cands[t]:
+            budget["nodes"] -= 1
+            assignment[t] = v
+            if consistent() and dfs(i + 1):
+                return True
+        assignment.pop(t, None)
+        return False
+
+    if dfs(0) and budget["nodes"] > 0:
+        sys_ = NumeralSystem(assignment)
+        # verify every attested equation round-trips
+        if all(sys_.text_to_num(p) == v for p, v in attested):
+            return sys_
+    return None
+
+
+def _divisors(n: int) -> List[int]:
+    n = abs(n)
+    out = []
+    for d in range(1, int(n ** 0.5) + 1):
+        if n % d == 0:
+            out += [d, n // d]
+    return sorted(set(out))
+
+
+def extract_attested(pairs) -> List[Tuple[str, int]]:
+    """From preprocess Pairs, pull (phrase, int) where one side is a number."""
+    out = []
+    for p in pairs:
+        for a, b in ((p.src, p.tgt), (p.tgt, p.src)):
+            bs = b.strip().replace(",", "").replace(" ", "")
+            if re.fullmatch(r"\d+", bs):
+                out.append((a, int(bs)))
+                break
+    return out

solver/pipeline.py

@@ -0,0 +1,382 @@
+"""Pipeline (shared by script.py and eval/dev_harness.py): symbolic pass, then
+LLM answering (lean or scaffolded), merge, never-empty guarantee."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional, Sequence
+
+from .budget import Budget
+from .direct import align_answers, solve_direct
+from .llm import LLMClient, NullClient
+from .matching import solve_match_letters_llm
+from .preprocess import Puzzle, parse_puzzle
+from .router import solve_puzzle_ex
+from .scaffold import build_scaffold, light_hint
+
+CONF_KEEP = 0.5  # symbolic answers at/above this verifier fit override the LLM
+LLM_BATCH = 4  # puzzles per generation batch (T4 KV-cache friendly)
+DEEP_MODE_MAX_ROWS = 30  # at/below this many puzzles, spend more per puzzle
+
+_DIGITS_RX = re.compile(r"\d[\d,. ]*")
+_LETTER_RX = re.compile(r"\b([A-Z])\b")
+_PREAMBLE_RX = re.compile(
+    r"^(?:the\s+)?(?:answer|translation|result)\s*(?:is|:)\s*", re.IGNORECASE)
+_TERMINAL_PUNCT = (".", "!", "?")
+
+
+@dataclass
+class PuzzleResult:
+    row_id: str
+    answers: List[str]
+    explanation: str = ""
+    confs: List[float] = field(default_factory=list)
+    methods: List[str] = field(default_factory=list)
+    llm_used: bool = False
+    raw: str = ""
+
+
+def clean_llm_answer(ans: str, task_type: str, option_labels: Sequence[str] = ()) -> str:
+    """Per-task-type format guard on a parsed LLM answer line. Conservative:
+    only rewrites when the expected shape is unambiguous."""
+    a = _PREAMBLE_RX.sub("", ans.strip()).strip().strip("'\"“”")
+    if task_type == "text_to_num":
+        m = _DIGITS_RX.search(a)
+        if m:
+            digits = re.sub(r"[,. ]", "", m.group(0))
+            if digits.isdigit():
+                return digits
+    elif task_type == "match_letters":
+        if len(a) > 2:  # "B. the bird sleeps" or "option B" -> "B"
+            candidates = _LETTER_RX.findall(a)
+            wanted = [c for c in candidates if not option_labels or c in option_labels]
+            if len(set(wanted)) == 1:
+                return wanted[0]
+    return a
+
+
+def _vote(cands: Sequence[Optional[str]], anchor: Optional[str]) -> Optional[str]:
+    """Greedy-anchored vote: keep `anchor` (the greedy answer) unless at least
+    two sampled candidates agree on the same normalised form AND that form
+    outnumbers the anchor's support. Monotone — it can only fire on genuine
+    agreement, so it never replaces greedy with a lone sample."""
+    from collections import Counter
+    cands = [c for c in cands if c and str(c).strip()]
+    if anchor is None:
+        anchor = cands[0] if cands else None
+    if len(cands) < 3:
+        return anchor
+    norm = lambda s: re.sub(r"\s+", " ", str(s).strip().lower())
+    groups: dict = {}
+    for c in cands:
+        groups.setdefault(norm(c), []).append(c)
+    anchor_support = len(groups.get(norm(anchor), [])) if anchor else 0
+    best_key = max(groups, key=lambda k: len(groups[k]))
+    if len(groups[best_key]) >= 2 and len(groups[best_key]) > anchor_support:
+        return Counter(groups[best_key]).most_common(1)[0][0]
+    return anchor
+
+
+def induce_format(answer: str, puzzle: Puzzle, item_idx: int) -> str:
+    """Nudge an answer toward the dataset's surface convention (fable §3):
+    if the attested answers on this item's side overwhelmingly end in a
+    terminal punctuation mark or start with a capital, mirror that. Converts
+    chrF-close answers into EM hits, which the geometric-mean scoring rewards
+    twice. Conservative: only fires on a near-unanimous (>=85%) convention,
+    only ADDS a missing terminal mark or leading capital, never strips."""
+    if puzzle.task_type not in ("translation", "fill_blanks"):
+        return answer
+    a = answer.strip()
+    if not a:
+        return answer
+    it = puzzle.items[item_idx] if item_idx < len(puzzle.items) else None
+    direction = getattr(it, "direction", None)
+    # answer side: to_work -> work-language (tgt); else task-language (src)
+    side = [p.tgt for p in puzzle.pairs] if direction == "to_work" \
+        else [p.src for p in puzzle.pairs]
+    side = [s.strip() for s in side if s and s.strip()]
+    if len(side) < 4:
+        return answer
+    n = len(side)
+    # terminal punctuation: only if a single mark dominates
+    for mark in _TERMINAL_PUNCT:
+        if sum(1 for s in side if s.endswith(mark)) / n >= 0.85:
+            if not a.endswith(_TERMINAL_PUNCT):
+                a = a + mark
+            break
+    # leading capitalization
+    if sum(1 for s in side if s[:1].isupper()) / n >= 0.85:
+        if a[:1].islower():
+            a = a[:1].upper() + a[1:]
+    return a
+
+
+def _symbolic_explanation(puzzle: Puzzle, methods: Sequence[str]) -> str:
+    used = [m for m in dict.fromkeys(methods) if m not in ("none", "fallback")]
+    if not used:
+        return ("- Answered by nearest-attested analogy over the given examples "
+                "(no reliable rule could be verified).")
+    tmpl = {
+        "numeral system (verified)": (
+            "- Induced each morpheme's numeric value from the attested numerals "
+            "and verified the system reproduces every given example; applied it "
+            "to each query item (smaller-before-larger multiplies, otherwise "
+            "values add)."),
+        "table completion": (
+            "- Learned the mapping between the paradigm-table columns from the "
+            "attested rows (leave-one-out verified) and applied it to each "
+            "incomplete row."),
+        "template substitution": (
+            "- For each query, took the closest attested sentence and swapped "
+            "the differing words through morpheme alignments induced from "
+            "minimal pairs in the data."),
+        "induced grammar": (
+            "- Induced a lexicon and affix rules that reproduce the attested "
+            "pairs exactly, then applied them mechanically to the query items."),
+        "optimal matching": (
+            "- Scored every form-meaning pair by shared-morpheme/shared-word "
+            "consistency and picked the globally optimal assignment."),
+    }
+    return "\n".join(tmpl.get(m, f"- Solved by {m}.") for m in used)
+
+
+def run_pipeline(rows: Sequence[dict], client: Optional[LLMClient] = None,
+                 budget: Optional[Budget] = None, verbose: bool = True,
+                 conf_keep: float = CONF_KEEP, llm_batch: int = LLM_BATCH,
+                 max_new_tokens: Optional[int] = None,
+                 checkpoint: Optional[Callable[[List["PuzzleResult"]], None]] = None,
+                 lean: bool = False,
+                 use_match_assignment: bool = True,
+                 vote_samples: int = 0,
+                 vote_temp: float = 0.5,
+                 hint: bool = False
+                 ) -> List[PuzzleResult]:
+    """`checkpoint`, when given, is called with the (complete, valid) results
+    after the symbolic pass and after every LLM batch — so a crash at ANY
+    later point still leaves a full submission on disk."""
+    client = client or NullClient()
+    budget = budget or Budget()
+
+    # adaptive mode: the hidden test is small (an IOL contest reformatted into
+    # a handful of multi-item rows), so default to spending more per puzzle.
+    # A large row count flips us to coverage mode (shorter generations, serve
+    # the most puzzles).
+    deep_mode = len(rows) <= DEEP_MODE_MAX_ROWS
+    if max_new_tokens is None:
+        max_new_tokens = 2048 if deep_mode else 1024
+
+    def log(msg: str) -> None:
+        if verbose:
+            print(msg, flush=True)
+
+    def save() -> None:
+        if checkpoint is not None:
+            try:
+                checkpoint(results)
+            except Exception:
+                pass
+
+    # ---- 1. symbolic pass ----
+    results: List[PuzzleResult] = []
+    puzzles: List[Optional[Puzzle]] = []
+    for i, row in enumerate(rows):
+        rid = str(row.get("id", i))
+        try:
+            p = parse_puzzle(row)
+            answers, confs, methods = solve_puzzle_ex(p, NullClient(), budget,
+                                                      puzzles_left=len(rows) - i)
+        except Exception:
+            p = None
+            answers, confs, methods = [str(row.get("query", "?")).strip() or "?"], [0.0], ["fallback"]
+        puzzles.append(p)
+        results.append(PuzzleResult(rid, answers, "", confs, methods))
+    # make every result submission-valid NOW (explanations + non-empty), so
+    # each checkpoint from here on is a complete fallback submission
+    for i, r in enumerate(results):
+        r.answers = [str(a).strip() or "?" for a in r.answers]
+        r.explanation = (_symbolic_explanation(puzzles[i], r.methods)
+                         if puzzles[i] is not None else
+                         "- No parseable structure found; answered by "
+                         "closest-example analogy.")
+    log(f"symbolic pass done in {budget.elapsed():.1f}s")
+    save()
+
+    # arm the per-token wall-clock abort so generation can't overrun the budget
+    try:
+        client.deadline = budget.start + budget.total - budget.safety
+    except Exception:
+        pass
+
+    # ---- 1b. match_letters assignment pass (model logprobs -> Hungarian) ----
+    # Free-form generation answers match_letters with the identity permutation
+    # (scores ~0). Solve it as an assignment from the model's own distribution
+    # instead. Solved puzzles get high confidence so the free-form LLM pass
+    # skips them; a declined puzzle falls through to that pass unchanged.
+    # Gated by use_match_assignment: when off, match_letters puzzles go through
+    # the normal free-form LLM pass.
+    if not use_match_assignment:
+        log("match_letters assignment pass disabled; using free-form LLM path")
+    if use_match_assignment and getattr(client, "can_score", False):
+        n_assigned = 0
+        for i, p in enumerate(puzzles):
+            if p is None or p.task_type != "match_letters" or budget.exhausted():
+                continue
+            try:
+                letters = solve_match_letters_llm(p, client)
+            except Exception as e:
+                log(f"  match_letters solver failed on {results[i].row_id}: "
+                    f"{type(e).__name__}: {e}")
+                letters = None
+            r = results[i]
+            if letters and len(letters) == len(r.answers):
+                r.answers = [str(x).strip() or "?" for x in letters]
+                r.confs = [0.9] * len(letters)
+                r.methods = ["llm-assignment"] * len(letters)
+                r.llm_used = True
+                n_assigned += 1
+        if n_assigned:
+            log(f"match_letters assignment solver used on {n_assigned} puzzle(s)")
+            save()
+
+    # ---- 2. scaffolded LLM pass ----
+    # In lean mode symbolic is a pure fallback: the model answers EVERY puzzle
+    # and its answer wins wherever it produced one (symbolic stands only for the
+    # items it left blank). Otherwise the model runs only on low-confidence
+    # puzzles and verified symbolic answers override it.
+    if lean:
+        need = [i for i in range(len(results)) if puzzles[i] is not None]
+    else:
+        need = [i for i, r in enumerate(results)
+                if puzzles[i] is not None and any(c < conf_keep for c in r.confs)]
+    if deep_mode:
+        # weakest-first: a budget cutoff then drops the puzzles we could help least
+        need.sort(key=lambda i: sum(min(c, conf_keep) for c in results[i].confs)
+                  / max(len(results[i].confs), 1))
+    else:
+        # coverage: shortest prompt first maximizes puzzles served per second
+        need.sort(key=lambda i: len(puzzles[i].context) + len(puzzles[i].query))
+    from .direct import LEAN_SYSTEM, SYSTEM
+    sys_prompt = LEAN_SYSTEM if lean else SYSTEM
+    log(f"LLM pass ({'lean' if lean else 'scaffold'}, "
+        f"{'deep' if deep_mode else 'coverage'}, "
+        f"max_new_tokens={max_new_tokens}): {len(need)}/{len(rows)} puzzles "
+        f"need the model; client={'yes' if client.available else 'no'}")
+    parse_ok = parse_fail = 0
+    if client.available and need:
+        done = 0
+        for start in range(0, len(need), llm_batch):
+            if budget.exhausted():
+                log(f"budget cutoff after {done} LLM puzzles")
+                break
+            batch = need[start : start + llm_batch]
+            if lean:
+                # lean: no scaffold, unless the optional light hint is enabled
+                scaffolds = [light_hint(puzzles[i]) if hint else "" for i in batch]
+            else:
+                scaffolds = []
+                for i in batch:
+                    p, r = puzzles[i], results[i]
+                    try:
+                        scaffolds.append(build_scaffold(p, r.answers, r.confs, r.methods))
+                    except Exception:
+                        scaffolds.append("")
+            try:
+                outs = solve_direct([puzzles[i] for i in batch], client, scaffolds,
+                                    max_new_tokens=max_new_tokens, system=sys_prompt,
+                                    lean=lean)
+            except Exception as e:
+                # systemic generation failure (bad load, driver, etc.) — the
+                # symbolic answers already on every item are the submission
+                log(f"LLM batch failed ({type(e).__name__}: {e}); "
+                    f"keeping symbolic answers for the rest")
+                break
+            for i, (direct, expl, raw, found) in zip(batch, outs):
+                r = results[i]
+                p = puzzles[i]
+                r.llm_used = True
+                r.raw = raw
+                if found:
+                    parse_ok += 1
+                else:
+                    # no answer block parsed even after salvage: do NOT let
+                    # reasoning prose overwrite the symbolic answers
+                    parse_fail += 1
+                    continue
+                labels = sorted(p.lettered) if p.lettered else ()
+                aligned = align_answers(direct, len(r.answers))
+                for j, d in enumerate(aligned):
+                    # lean: the model's answer wins wherever it gave one;
+                    # scaffold: only where symbolic isn't confident
+                    if d and (lean or r.confs[j] < conf_keep):
+                        cleaned = clean_llm_answer(d, p.task_type, labels)
+                        if cleaned:
+                            # lean mode ships the model's answer as-is (no
+                            # punctuation/casing induction)
+                            if not lean:
+                                cleaned = induce_format(cleaned, p, j)
+                            r.answers[j] = cleaned.strip() or r.answers[j]
+                            r.methods[j] = "llm"
+                if expl:
+                    r.explanation = expl
+            done += len(batch)
+            log(f"  llm {done}/{len(need)}  ok={parse_ok} fail={parse_fail}  "
+                f"t={budget.elapsed():.0f}s")
+            save()
+        log(f"LLM pass done: {parse_ok} parsed, {parse_fail} unparsable "
+            f"(kept symbolic); {budget.elapsed():.0f}s elapsed")
+
+    # ---- 2b. light greedy-anchored self-consistency voting (lean only) ----
+    # The greedy answers are already checkpointed; sampled passes can only
+    # displace an item on genuine agreement (see _vote), so this is monotone and
+    # budget-gated — if the clock runs out we simply keep the greedy answers.
+    if lean and vote_samples > 0 and client.available and need and not budget.exhausted():
+        ballots = {i: [list(results[i].answers)] for i in need}  # greedy = ballot 0
+        done_votes = 0
+        for _s in range(vote_samples):
+            if budget.exhausted():
+                break
+            failed = False
+            for start in range(0, len(need), llm_batch):
+                if budget.exhausted():
+                    break
+                batch = need[start : start + llm_batch]
+                try:
+                    outs = solve_direct([puzzles[i] for i in batch], client,
+                                        ["" for _ in batch],
+                                        max_new_tokens=max_new_tokens,
+                                        system=sys_prompt, lean=True,
+                                        sample=True, temperature=vote_temp)
+                except Exception as e:
+                    log(f"vote pass failed ({type(e).__name__}: {e}); "
+                        f"keeping greedy answers")
+                    failed = True
+                    break
+                for i, (direct, _e, _raw, _f) in zip(batch, outs):
+                    p = puzzles[i]
+                    labels = sorted(p.lettered) if p.lettered else ()
+                    ballot = []
+                    for j in range(len(results[i].answers)):
+                        d = direct[j] if j < len(direct) else None
+                        ballot.append(clean_llm_answer(d, p.task_type, labels)
+                                      if d else None)
+                    ballots[i].append(ballot)
+            if failed:
+                break
+            done_votes += 1
+            for i in need:  # re-vote (greedy-anchored) after each sample pass
+                r = results[i]
+                anchor = ballots[i][0]
+                for j in range(len(r.answers)):
+                    v = _vote([b[j] for b in ballots[i] if j < len(b)],
+                              anchor[j] if j < len(anchor) else None)
+                    if v and str(v).strip():
+                        r.answers[j] = str(v).strip()
+            save()
+        log(f"voting: {done_votes}/{vote_samples} sample pass(es); "
+            f"t={budget.elapsed():.0f}s")
+
+    # ---- 3. final never-empty guarantee ----
+    for r in results:
+        r.answers = [str(a).strip() or "?" for a in r.answers]
+    return results

solver/preprocess.py

@@ -0,0 +1,384 @@
+"""Parse Linguini puzzles: normalize text, parse the context (pipe tables,
+numbered/lettered lists, pairs) and the query into answerable items."""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+_ITEM_PREFIX = re.compile(r"^\s*\(?(\d{1,3})[\.\)]\s+")
+_LETTER_PREFIX = re.compile(r"^\s*\(?([A-Z])[\.\)]\s+")
+_BLANK_MARK = re.compile(r"\((\d{1,3})\)")
+_BLANK_LINE = re.compile(r"_{2,}|…|\.{4,}")
+
+# non-pipe two-side separators, tried in order on non-table lines
+_SEPARATORS = [
+    ("tab", re.compile(r"\t+")),
+    ("equals", re.compile(r"\s+=\s+")),
+    ("emdash", re.compile(r"\s+—\s+")),
+    ("endash", re.compile(r"\s+–\s+")),
+    ("arrow", re.compile(r"\s*(?:->|→)\s*")),
+    ("hyphen", re.compile(r"\s+-\s+")),
+    ("means", re.compile(r"\s+means\s+", re.IGNORECASE)),
+]
+
+# common work-language names (queries say "Translate into English:")
+_WORK_LANG_NAMES = {
+    "eng": "english", "fra": "french", "spa": "spanish", "por": "portuguese",
+    "rus": "russian", "deu": "german",
+}
+
+
+def normalize(text: str) -> str:
+    """NFC-normalize, unify exotic whitespace/quotes. Keeps diacritics, tone
+    marks, case, and punctuation (EM comparison is punctuation-sensitive)."""
+    if text is None:
+        return ""
+    t = unicodedata.normalize("NFC", str(text))
+    t = t.replace(" ", " ")
+    t = re.sub(r"[ \t]+", " ", t)
+    return t.strip()
+
+
+def tokenize(s: str) -> List[str]:
+    """Unicode word tokenization. Keeps combining marks, word-internal
+    apostrophes/hyphens, and subscript/superscript markers (tone letters,
+    person markers like you_{sg})."""
+    s = normalize(s)
+    return re.findall(r"[^\s,;.!?()\[\]\"«»|]+", s)
+
+
+def strip_punct(tok: str) -> str:
+    return tok.strip(",;.!?()[]\"«»").strip()
+
+
+@dataclass
+class Pair:
+    src: str  # task-language side by convention
+    tgt: str  # work-language side (gloss/translation)
+    sep: str = ""
+    line_no: int = -1
+    label: str = ""  # numbered prefix if the line carried one
+
+
+@dataclass
+class QueryItem:
+    number: str  # label as it appeared ("17", "3", "") — "" for bare lines
+    text: str
+    direction: Optional[str] = None  # "to_task" | "to_work" | None
+    has_blank: bool = False
+    row: Optional[List[str]] = None  # for table-blank items: full row cells
+    blank_col: Optional[int] = None  # which cell holds this item's (k) marker
+
+
+@dataclass
+class Puzzle:
+    id: str
+    context: str
+    query: str
+    work_lang: str = ""
+    task_lang: str = ""
+    task_type: str = ""
+    eval_type: str = ""
+    pairs: List[Pair] = field(default_factory=list)
+    items: List[QueryItem] = field(default_factory=list)
+    hints: List[str] = field(default_factory=list)
+    tables: List[List[List[str]]] = field(default_factory=list)  # blocks of rows of cells
+    numbered: Dict[str, str] = field(default_factory=dict)  # "1" -> form (list contexts)
+    lettered: Dict[str, str] = field(default_factory=dict)  # "A" -> meaning
+
+
+def _split_cells(line: str) -> List[str]:
+    return [c.strip() for c in line.split("|")]
+
+
+def _strip_item_prefix(line: str) -> Tuple[str, str]:
+    """Returns (label, rest). Label may be a number or capital letter."""
+    m = _ITEM_PREFIX.match(line)
+    if m:
+        return m.group(1), line[m.end():].strip()
+    m = _LETTER_PREFIX.match(line)
+    if m:
+        return m.group(1), line[m.end():].strip()
+    return "", line.strip()
+
+
+def _looks_header(cells: List[str]) -> bool:
+    """A table header names languages/columns: 'Proto-Chamic | Tsat | meaning'."""
+    if len(cells) < 2:
+        return False
+    tail = cells[-1].lower()
+    if tail in ("meaning", "meanings", "translation", "translations", "english",
+                "value", "values", "gloss"):
+        return True
+    # all cells capitalized single-ish words with no digits — likely names
+    ok = 0
+    for c in cells:
+        if c and not any(ch.isdigit() for ch in c) and c[0].isupper() and len(c.split()) <= 3:
+            ok += 1
+    return ok == len(cells) and len(cells) >= 3
+
+
+def parse_context(ctx: str) -> Tuple[List[Pair], List[str], List[List[List[str]]], Dict[str, str], Dict[str, str]]:
+    """Parse context into (pairs, hints, tables, numbered, lettered)."""
+    pairs: List[Pair] = []
+    hints: List[str] = []
+    tables: List[List[List[str]]] = []
+    numbered: Dict[str, str] = {}
+    lettered: Dict[str, str] = {}
+
+    cur_table: List[List[str]] = []
+    for i, raw in enumerate(str(ctx).splitlines()):
+        line = normalize(raw)
+        if not line:
+            if cur_table:
+                tables.append(cur_table)
+                cur_table = []
+            continue
+        label, body = _strip_item_prefix(line)
+
+        if "|" in body:
+            cells = _split_cells(body)
+            if _looks_header(cells) and not cur_table:
+                hints.append(line)
+                continue
+            cur_table.append(cells)
+            has_blank = bool(_BLANK_MARK.search(body))
+            if len(cells) >= 2 and cells[0] and cells[-1] and not has_blank:
+                pairs.append(Pair(src=cells[0], tgt=cells[-1], sep="pipe",
+                                  line_no=i, label=label))
+            if label and not has_blank:
+                numbered[label] = cells[0]
+            continue
+
+        if cur_table:
+            tables.append(cur_table)
+            cur_table = []
+
+        # non-pipe separators (= , — , tab ...)
+        matched = False
+        for name, rx in _SEPARATORS:
+            parts = rx.split(body, maxsplit=1)
+            if len(parts) == 2 and parts[0].strip() and parts[1].strip():
+                pairs.append(Pair(src=parts[0].strip(), tgt=parts[1].strip(),
+                                  sep=name, line_no=i, label=label))
+                if label:
+                    # the full line is the referable entry ("equalities (1-9)")
+                    numbered[label] = body
+                matched = True
+                break
+        if matched:
+            continue
+
+        # single-column list entries (match_letters forms/meanings)
+        if label:
+            if label.isdigit():
+                numbered[label] = body
+            else:
+                lettered[label] = body
+            continue
+
+        hints.append(line)
+
+    if cur_table:
+        tables.append(cur_table)
+    return pairs, hints, tables, numbered, lettered
+
+
+_INSTRUCTION_VERBS = (
+    r"(translate|fill|write|spell|determine|give|complete|convert|match|answer|"
+    r"say|pair|transcribe|provide|express|render|decipher|find|identify|"
+    r"choose|select|here|below|these|the following)"
+)
+_INSTRUCTION_RX = re.compile(r"^" + _INSTRUCTION_VERBS + r"\b", re.IGNORECASE)
+_INSTRUCTION_ANY_RX = re.compile(r"\b" + _INSTRUCTION_VERBS + r"\b", re.IGNORECASE)
+
+
+def _is_instruction(line: str) -> bool:
+    """Instruction lines are work-language imperatives ("Translate into X:").
+    Matching is verb-anchored — a bare trailing colon is NOT enough, because
+    task-language forms can end in ':' (length marks: "si teŋku bugdiŋi:").
+    A line that ends with ':' AND contains an instruction verb anywhere is
+    also an instruction ("In Drehu tusi is 'book'. Translate from Drehu:")."""
+    line = (line or "").strip()
+    if not line:
+        return False
+    if _BLANK_MARK.search(line) or "|" in line:
+        return False
+    if _INSTRUCTION_RX.match(line):
+        return True
+    return line.endswith(":") and bool(_INSTRUCTION_ANY_RX.search(line))
+
+
+def parse_query(query: str) -> Tuple[List[QueryItem], List[str]]:
+    """Split query into answerable items + instruction lines.
+
+    Item sources, in the order encountered:
+    - (k)-markers inside lines (usually pipe rows): one item per marker, with
+      the row cells and blank column recorded;
+    - numbered lines "17. ..." (numbering may continue the context's);
+    - bare non-instruction lines: one item per line.
+    """
+    text = str(query or "")
+    items: List[QueryItem] = []
+    instructions: List[str] = []
+    _TERMINAL = (".", "!", "?", ":", ";", '"', "”", "’")
+
+    for raw in text.splitlines():
+        line = normalize(raw)
+        if not line:
+            continue
+        marks = _BLANK_MARK.findall(line)
+        if marks:
+            cells = _split_cells(line) if "|" in line else [line]
+            for k in marks:
+                blank_col = next(
+                    (ci for ci, c in enumerate(cells) if f"({k})" in c), None)
+                items.append(QueryItem(
+                    number=k, text=line, has_blank=True,
+                    row=cells if len(cells) > 1 else None, blank_col=blank_col))
+            continue
+        if "|" in line:
+            label, body = _strip_item_prefix(line)
+            if label:
+                # numbered table row = one item; the answer fills whichever
+                # column the context table has that this row lacks
+                items.append(QueryItem(number=label, text=body,
+                                       row=_split_cells(body)))
+            else:
+                instructions.append(line)  # header/echo row
+            continue
+        label, body = _strip_item_prefix(line)
+        if label:
+            items.append(QueryItem(number=label, text=body,
+                                   has_blank=bool(_BLANK_LINE.search(body))))
+            continue
+        if _is_instruction(line):
+            instructions.append(line)
+            continue
+        if items and items[-1].number and not items[-1].text.rstrip().endswith(_TERMINAL):
+            items[-1].text += " " + line  # wrapped continuation of a numbered item
+            continue
+        items.append(QueryItem(number="", text=line,
+                               has_blank=bool(_BLANK_LINE.search(line))))
+
+    # when the query has numbered items, stray unnumbered lines around them
+    # are notes ("spoken on Bvuŋkaden"), not answerable items
+    if any(it.number for it in items):
+        items = [it for it in items if it.number]
+    # items with numeric labels answer in label order when labels are complete
+    if items and all(it.number.isdigit() for it in items):
+        items.sort(key=lambda it: int(it.number))
+    return items, instructions
+
+
+def detect_direction(item_text: str, task_material: str, work_material: str,
+                     instructions: List[str], work_lang: str) -> str:
+    """Per-item direction: does the answer belong to the task language
+    ('to_task') or the work language ('to_work')?
+
+    1. Explicit instruction: "into English" (work-lang name) vs "into X".
+    2. Script similarity: if the item text overlaps the task-language material
+       character-wise, it is task-language text needing analysis (to_work).
+    """
+    joined = (" ".join(instructions) + " " + item_text).lower()
+    wl_name = _WORK_LANG_NAMES.get(work_lang.split("_")[0][:3].lower(), "")
+    m = re.search(r"(?:into|in|to)\s+(?:the\s+)?([A-Za-zÀ-ž’' -]{2,30}?)\s*(?:language)?\s*[:.]", joined + ":")
+    if m:
+        named = m.group(1).strip().lower()
+        if wl_name and wl_name in named:
+            return "to_work"
+        if named and not any(w in named for w in ("digit", "numeral", "number", "blank")):
+            return "to_task"
+    sim_task = _char_overlap(item_text, task_material)
+    sim_work = _char_overlap(item_text, work_material)
+    return "to_work" if sim_task >= sim_work else "to_task"
+
+
+def _char_overlap(s: str, material: str, n: int = 3) -> float:
+    s_ = "".join(s.lower().split())
+    m_ = "".join(material.lower().split())
+    if len(s_) < n or len(m_) < n:
+        return 0.0
+    grams = {s_[i : i + n] for i in range(len(s_) - n + 1)}
+    hits = sum(1 for g in grams if g in m_)
+    return hits / len(grams)
+
+
+_RANGE_RX = re.compile(r"\((\d{1,3})\s*[–—-]\s*(\d{1,3})\)")
+
+
+def _items_from_context(p: Puzzle) -> List[QueryItem]:
+    """When the query is instruction-only ("Fill in the blanks (1–14)",
+    "Determine the correct correspondences", "Write the equalities (1–9) in
+    numerals"), the answerable items live in the CONTEXT: (k) blank markers,
+    or the numbered list entries. Last resort: the query itself is one item."""
+    rng = _RANGE_RX.search(p.query or "")
+    lo, hi = (int(rng.group(1)), int(rng.group(2))) if rng else (None, None)
+
+    def in_range(k: str) -> bool:
+        return lo is None or (k.isdigit() and lo <= int(k) <= hi)
+
+    ctx_blanks: List[QueryItem] = []
+    for raw in str(p.context).splitlines():
+        line = normalize(raw)
+        for k in _BLANK_MARK.findall(line):
+            if not in_range(k):
+                continue
+            cells = _split_cells(line) if "|" in line else [line]
+            blank_col = next((ci for ci, c in enumerate(cells) if f"({k})" in c), None)
+            ctx_blanks.append(QueryItem(number=k, text=line, has_blank=True,
+                                        row=cells if len(cells) > 1 else None,
+                                        blank_col=blank_col))
+    if ctx_blanks:
+        ctx_blanks.sort(key=lambda it: int(it.number))
+        return ctx_blanks
+
+    if p.numbered and (p.task_type == "match_letters" or rng or p.lettered):
+        keys = sorted((k for k in p.numbered if in_range(k)), key=int)
+        if keys:
+            return [QueryItem(number=k, text=p.numbered[k]) for k in keys]
+
+    q = normalize(p.query)
+    return [QueryItem(number="", text=q)] if q else []
+
+
+def parse_puzzle(row: dict) -> Puzzle:
+    """Build a Puzzle from a CSV/dataset row (id, context, query, work_lang,
+    task_lang, task_type, eval_type)."""
+    ctx = str(row.get("context", "") or "")
+    p = Puzzle(
+        id=str(row.get("id", "")),
+        context=ctx,
+        query=str(row.get("query", "") or ""),
+        work_lang=str(row.get("work_lang", "") or ""),
+        task_lang=str(row.get("task_lang", "") or ""),
+        task_type=str(row.get("task_type", "") or "").strip().lower(),
+        eval_type=str(row.get("eval_type", "") or ""),
+    )
+    p.pairs, p.hints, p.tables, p.numbered, p.lettered = parse_context(ctx)
+    p.items, instructions = parse_query(p.query)
+    p.hints.extend(instructions)
+
+    # letter-labelled query entries are answer OPTIONS when digit-labelled
+    # items coexist (match tasks list both: "19. form ... S. meaning")
+    digit_items = [it for it in p.items if it.number.isdigit()]
+    letter_items = [it for it in p.items if it.number and not it.number.isdigit()]
+    if digit_items and letter_items:
+        for it in letter_items:
+            p.lettered[it.number] = it.text
+        p.items = digit_items
+
+    if not p.items:
+        p.items = _items_from_context(p)
+
+    task_material = " ".join(x.src for x in p.pairs) + " " + " ".join(p.numbered.values())
+    work_material = " ".join(x.tgt for x in p.pairs) + " " + " ".join(p.lettered.values())
+    for it in p.items:
+        if it.row is not None:
+            continue  # table-blank items get direction from their row in the router
+        it.direction = detect_direction(it.text, task_material, work_material,
+                                        instructions, p.work_lang)
+    return p

solver/router.py

@@ -0,0 +1,226 @@
+"""Router: dispatch each puzzle to its symbolic solver (numerals, matching,
+tables, translation) and return one answer per item. Never raises or empties."""
+
+from __future__ import annotations
+
+import re
+from typing import Callable, List, Optional, Tuple
+
+from .budget import Budget
+from .fallback import ensure_nonempty, fallback_answer
+from .llm import LLMClient, NullClient
+from .matching import solve_matching
+from .numerals import extract_attested, induce
+from .preprocess import Pair, Puzzle, QueryItem, normalize
+from .synth import synthesize
+from .tables import TableSolver
+from .template import TemplateTranslator
+from .verifier import evaluate, leave_one_out
+
+_NUM_RX = re.compile(r"\d+")
+_QUOTED = re.compile(r"[\"«]([^\"»]+)[\"»]|‘([^’]{2,})’|'([^']{2,})'")
+
+
+def _payload(text: str) -> str:
+    """Payload of a whole-query item: quoted material, else text after a
+    colon, else the final word of an instruction-like sentence."""
+    t = normalize(text)
+    m = _QUOTED.search(t)
+    if m:
+        return next(g for g in m.groups() if g).strip()
+    if ":" in t:
+        tail = t.split(":", 1)[1].strip()
+        if tail:
+            return tail
+    m2 = re.match(r"^(give|translate|write|say|transcribe)\b.*\b(?:word|numeral|phrase|form)\s+(\S+)\s*$",
+                  t, re.IGNORECASE)
+    if m2:
+        return m2.group(2).strip(".?!")
+    return t
+
+
+def solve_puzzle(puzzle: Puzzle, client: Optional[LLMClient] = None,
+                 budget: Optional[Budget] = None, puzzles_left: int = 1) -> List[str]:
+    return solve_puzzle_ex(puzzle, client, budget, puzzles_left)[0]
+
+
+def solve_puzzle_ex(puzzle: Puzzle, client: Optional[LLMClient] = None,
+                    budget: Optional[Budget] = None, puzzles_left: int = 1
+                    ) -> Tuple[List[str], List[float], List[str]]:
+    """Returns (answers, confidences, methods). Confidence is the verifier
+    evidence behind each answer (LOO/eval fit of the solver that produced it,
+    or ~1.0 for round-trip-verified numeral systems); 0.0 marks answers that
+    came from the never-empty fallback ladder — those are the items worth LLM
+    budget. Methods name the producing solver, for prompt candidate blocks
+    and explanation-track traces."""
+    client = client or NullClient()
+    budget = budget or Budget()
+    items = puzzle.items or [QueryItem(number="", text=puzzle.query or "")]
+    try:
+        answers, confs, methods = _dispatch(puzzle, items, client, budget, puzzles_left)
+    except Exception:
+        answers, confs, methods = None, None, None
+    if answers is None:
+        answers = [None] * len(items)
+    if confs is None:
+        confs = [0.0] * len(items)
+    if methods is None:
+        methods = ["none"] * len(items)
+    answers = (list(answers) + [None] * len(items))[: len(items)]
+    confs = (list(confs) + [0.0] * len(items))[: len(items)]
+    methods = (list(methods) + ["none"] * len(items))[: len(items)]
+    out = []
+    for i, (item, ans) in enumerate(zip(items, answers)):
+        if ans is None or not str(ans).strip():
+            confs[i] = 0.0
+            methods[i] = "fallback"
+        direction = item.direction or "to_work"
+        out.append(ensure_nonempty(ans, _payload(item.text), puzzle.pairs, direction))
+    return out, confs, methods
+
+
+def _dispatch(puzzle: Puzzle, items: List[QueryItem], client: LLMClient,
+              budget: Budget, left: int
+              ) -> Tuple[List[Optional[str]], List[float], List[str]]:
+    tt = puzzle.task_type
+    if tt in ("text_to_num", "num_to_text"):
+        return _solve_numerals(puzzle, items)
+    if tt == "match_letters":
+        return _solve_matching(puzzle, items)
+
+    table = TableSolver(puzzle)
+    answers: List[Optional[str]] = [None] * len(items)
+    confs: List[float] = [0.0] * len(items)
+    methods: List[str] = ["none"] * len(items)
+    plain_idx = []
+    for i, it in enumerate(items):
+        if it.row is not None and table.usable:
+            ans_conf = table.solve(it)
+            if ans_conf is not None:
+                answers[i], confs[i] = ans_conf
+                methods[i] = "table completion"
+        if answers[i] is None:
+            plain_idx.append(i)
+    if plain_idx:
+        translated, t_confs, t_methods = _solve_translation(
+            puzzle, [items[i] for i in plain_idx], client, budget, left)
+        for i, ans, c, m in zip(plain_idx, translated, t_confs, t_methods):
+            answers[i], confs[i], methods[i] = ans, c, m
+    return answers, confs, methods
+
+
+# ---------------------------------------------------------------- numerals
+
+def _solve_numerals(puzzle: Puzzle, items: List[QueryItem]
+                    ) -> Tuple[List[Optional[str]], List[float]]:
+    attested = extract_attested(puzzle.pairs)
+    system = induce(attested) if attested else None
+    out: List[Optional[str]] = []
+    for item in items:
+        text = _payload(item.text)
+        if puzzle.task_type == "text_to_num":
+            val = system.text_to_num(text) if system else None
+            out.append(str(val) if val is not None else None)
+        else:
+            m = _NUM_RX.search(text)
+            if system and m:
+                out.append(system.num_to_text(int(m.group(0)), [p for p, _ in attested]))
+            else:
+                out.append(None)
+    # an induced system is round-trip verified on every attested equation
+    confs = [0.9 if a is not None else 0.0 for a in out]
+    methods = ["numeral system (verified)" if a is not None else "none" for a in out]
+    return out, confs, methods
+
+
+# ---------------------------------------------------------------- matching
+
+def _solve_matching(puzzle: Puzzle, items: List[QueryItem]
+                    ) -> Tuple[List[Optional[str]], List[float]]:
+    """Items are forms; options are the lettered meanings (from context or
+    query). Answers are option letters when options exist, else the matched
+    meaning text."""
+    forms = [_payload(it.text) for it in items]
+    if puzzle.lettered:
+        labels = sorted(puzzle.lettered)
+        meanings = [puzzle.lettered[l] for l in labels]
+    else:
+        labels = None
+        meanings = [p.tgt for p in puzzle.pairs]
+    if not meanings:
+        return [None] * len(forms), [0.0] * len(forms), ["none"] * len(forms)
+    matched = dict(solve_matching(forms, meanings, puzzle.pairs))
+    out: List[Optional[str]] = []
+    for f in forms:
+        m = matched.get(f)
+        if m and labels:
+            out.append(labels[meanings.index(m)])
+        else:
+            out.append(m)
+    # Hungarian is optimal for its score matrix, but the matrix itself is only
+    # as good as the alignment evidence behind it — real-data EM is low, so
+    # this stays BELOW the pipeline's keep-threshold: it surfaces as a
+    # candidate hint in the LLM prompt rather than overriding the LLM
+    conf = 0.45 if puzzle.pairs else 0.25
+    return (out, [conf if a else 0.0 for a in out],
+            ["optimal matching" if a else "none" for a in out])
+
+
+# ------------------------------------------------------------- translation
+
+def _solve_translation(puzzle: Puzzle, items: List[QueryItem], client: LLMClient,
+                       budget: Budget, left: int
+                       ) -> Tuple[List[Optional[str]], List[float]]:
+    directions = {it.direction or "to_work" for it in items}
+    primary = "to_task" if "to_task" in directions else "to_work"
+    rounds = budget.cegis_rounds(left) if budget.allow_llm(left) else -1
+    synth_res = synthesize(puzzle, client, primary, rounds) if rounds >= 0 else None
+
+    solvers = {d: _pick_direction_solver(puzzle, synth_res, d) for d in directions}
+    answers, confs, methods = [], [], []
+    for it in items:
+        ans, conf, method = solvers[it.direction or "to_work"](_payload(it.text))
+        answers.append(ans)
+        confs.append(conf)
+        methods.append(method)
+    return answers, confs, methods
+
+
+def _pick_direction_solver(puzzle: Puzzle, synth_res, d: str) -> Callable[[str], Optional[str]]:
+    """Rank candidate solvers honestly and chain them (first non-None answer
+    wins). The grammar is a fixed program so it is evaluated directly on the
+    attested pairs (it must reproduce them); the template translator and the
+    fallback are *fit from* those pairs (they memorize them), so they are
+    scored leave-one-out — otherwise memorization would always beat a
+    generalizing grammar."""
+    attested = [(p.tgt, p.src) if d == "to_task" else (p.src, p.tgt) for p in puzzle.pairs]
+
+    def _subset(held_in_pairs) -> List[Pair]:
+        keep = set(held_in_pairs)
+        return [p for p in puzzle.pairs
+                if ((p.tgt, p.src) if d == "to_task" else (p.src, p.tgt)) in keep]
+
+    ranked: List[Tuple[float, int, str, Callable[[str], Optional[str]]]] = []
+
+    if synth_res and synth_res.interpreter:
+        fn = synth_res.interpreter.generate if d == "to_task" else synth_res.interpreter.analyze
+        v = evaluate(fn, attested, synth_res.grammar.mdl())
+        ranked.append((v.score, 0, "induced grammar", fn))
+
+    tmpl = TemplateTranslator(puzzle.pairs, d)
+    v_tmpl = leave_one_out(lambda held: TemplateTranslator(_subset(held), d).translate, attested)
+    ranked.append((v_tmpl.score, 1, "template substitution", tmpl.translate))
+
+    v_fb = leave_one_out(lambda held: (lambda q, kept=_subset(held): fallback_answer(q, kept, d)), attested)
+    ranked.append((v_fb.score, 2, "nearest attested", lambda q: fallback_answer(q, puzzle.pairs, d)))
+
+    ranked.sort(key=lambda t: (-t[0], t[1]))
+
+    def solve(q: str) -> Tuple[Optional[str], float, str]:
+        for score, _, name, fn in ranked:
+            ans = fn(q)
+            if ans:
+                return ans, max(score, 0.0), name
+        return None, 0.0, "none"
+
+    return solve

solver/scaffold.py

@@ -0,0 +1,113 @@
+"""Deterministic analysis blocks (segmentation, alignment, numerals) for the
+prompt: the full scaffold and the optional light hint."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional, Sequence, Tuple
+
+from .align import align as build_align
+from .numerals import extract_attested, induce
+from .preprocess import Puzzle, strip_punct, tokenize
+from .segment import Segmenter
+
+MAX_SEG_LINES = 40
+MAX_ALIGN_LINES = 30
+MAX_VOCAB = 60
+
+
+def analysis_blocks(puzzle: Puzzle) -> Tuple[str, str]:
+    """(segmentation block, alignment block) for the prompt. Also used by the
+    CEGIS proposer (synth.py)."""
+    amap = build_align(puzzle.pairs)
+    align_lines = []
+    for tok, cands in sorted(amap.items()):
+        top = [f"{t} ({s:.1f})" for t, s in cands[:2] if s > 0.2]
+        if top:
+            align_lines.append(f"  {tok} ~ {', '.join(top)}")
+
+    vocab, groups = [], {}
+    for p in puzzle.pairs:
+        for t in tokenize(p.src):
+            t = strip_punct(t).casefold()
+            if t and t not in vocab:
+                vocab.append(t)
+    vocab = vocab[:MAX_VOCAB]
+    for tok, cands in amap.items():
+        if cands:
+            groups.setdefault(cands[0][0], set()).add(tok)
+    seg = Segmenter().fit(vocab, share_groups=[g for g in groups.values() if len(g) > 1])
+    seg_lines = []
+    for w in vocab:
+        parts = seg.segment(w)
+        if len(parts) > 1:
+            seg_lines.append(f"  {w} = {'-'.join(parts)}")
+    return ("\n".join(seg_lines[:MAX_SEG_LINES]) or "  (none found)",
+            "\n".join(align_lines[:MAX_ALIGN_LINES]) or "  (none found)")
+
+
+def light_hint(puzzle: Puzzle, max_lines: int = 10) -> str:
+    """A minimal, optional hint for the lean prompt: a few morpheme segmentation
+    guesses, framed as fallible. Off by default; enabled via a toggle."""
+    try:
+        seg_block, _align = analysis_blocks(puzzle)
+    except Exception:
+        return ""
+    lines = [l for l in seg_block.splitlines() if l.strip() and "none found" not in l]
+    if not lines:
+        return ""
+    body = "\n".join(lines[:max_lines])
+    return ("Optional hint (an automatic guess at word parts; it may be wrong, "
+            "so rely on the data itself):\n" + body)
+
+
+def numeral_block(puzzle: Puzzle) -> str:
+    """Induced numeral-system values, when the CSP solved and round-trip
+    verified them — the strongest kind of hint we can give."""
+    if puzzle.task_type not in ("text_to_num", "num_to_text"):
+        return ""
+    attested = extract_attested(puzzle.pairs)
+    system = induce(attested) if attested else None
+    if system is None:
+        return ""
+    vals = ", ".join(f"{t}={v}" for t, v in sorted(system.values.items(), key=lambda kv: kv[1]))
+    return (f"Numeral analysis (verified against every attested example):\n  {vals}\n"
+            f"  combination rule: a smaller value directly before a larger one multiplies it; "
+            f"otherwise values add.")
+
+
+def candidate_block(items_answers: Sequence[Tuple[str, Optional[str], float, str]]) -> str:
+    """Symbolic candidate answers per item: (item label, answer, confidence,
+    method). Only candidates with real evidence are shown — a low-confidence
+    echo would anchor the model on garbage."""
+    lines = []
+    for label, ans, conf, method in items_answers:
+        if ans and conf >= 0.4:
+            lines.append(f"  item {label}: '{ans}'  (source: {method}, fit {conf:.2f})")
+    if not lines:
+        return ""
+    return ("Candidate answers from mechanical analysis (adopt if consistent with "
+            "the data, correct if not):\n" + "\n".join(lines))
+
+
+def build_scaffold(puzzle: Puzzle,
+                   answers: Optional[Sequence[Optional[str]]] = None,
+                   confs: Optional[Sequence[float]] = None,
+                   methods: Optional[Sequence[str]] = None) -> str:
+    """Full scaffold block for one puzzle's prompt."""
+    seg_block, align_block = analysis_blocks(puzzle)
+    parts = [
+        "## Mechanical analysis (computed from the data above; may contain errors — "
+        "the attested data always wins)",
+        f"Morpheme segmentation hypotheses:\n{seg_block}",
+        f"Word alignment hypotheses (task-language token ~ likely meaning):\n{align_block}",
+    ]
+    nb = numeral_block(puzzle)
+    if nb:
+        parts.append(nb)
+    if answers is not None and confs is not None:
+        labels = [it.number or str(i + 1) for i, it in enumerate(puzzle.items)]
+        meths = list(methods) if methods else ["symbolic"] * len(labels)
+        cb = candidate_block(list(zip(labels, answers, confs, meths)))
+        if cb:
+            parts.append(cb)
+    return "\n\n".join(parts)

solver/segment.py

@@ -0,0 +1,149 @@
+"""MDL-guided morpheme segmentation for tiny vocabularies, pure python.
+
+Greedy Morfessor-flavored search: start with whole words as morphs, repeatedly
+apply the single split that most reduces description length
+L(lexicon) + L(corpus | lexicon). Vocabularies here are tiny (10-100 word
+types), so an O(V * maxlen) sweep per iteration is instant.
+
+Alignment conditioning: tokens known (from align.py) to share a gloss get a
+bonus for splits that expose their shared substring — this is the
+"segmentation conditioned on alignment" step from the plan, and is what keeps
+MDL from over-segmenting on 20-word corpora.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Dict, Iterable, List, Optional, Sequence, Set, Tuple
+
+_MIN_MORPH = 1
+
+
+def _lex_cost(morphs: Iterable[str]) -> float:
+    # ~1 char = a few bits; +1 per morph for the boundary/index overhead
+    return sum(len(m) + 1 for m in set(morphs)) * 4.0
+
+
+def _corpus_cost(usage: Counter) -> float:
+    total = sum(usage.values())
+    if total == 0:
+        return 0.0
+    return -sum(c * math.log2(c / total) for c in usage.values())
+
+
+class Segmenter:
+    def __init__(self, share_bonus: float = 8.0):
+        self.share_bonus = share_bonus
+        self.seg: Dict[str, List[str]] = {}
+
+    def fit(
+        self,
+        words: Sequence[str],
+        counts: Optional[Counter] = None,
+        share_groups: Optional[List[Set[str]]] = None,
+        max_iters: int = 200,
+    ) -> "Segmenter":
+        """words: vocabulary (task-language word types).
+        counts: token frequencies (defaults to 1 each).
+        share_groups: sets of words believed to share a morpheme (same gloss
+        alignment); splits exposing a shared prefix/suffix get a bonus."""
+        counts = counts or Counter({w: 1 for w in words})
+        self.seg = {w: [w] for w in dict.fromkeys(words) if w}
+        shared_subs = self._shared_substrings(share_groups or [])
+
+        for _ in range(max_iters):
+            best = self._best_split(counts, shared_subs)
+            if best is None:
+                break
+            word, mi, cut = best
+            m = self.seg[word][mi]
+            self.seg[word][mi : mi + 1] = [m[:cut], m[cut:]]
+        return self
+
+    def _shared_substrings(self, groups: List[Set[str]]) -> Set[str]:
+        subs: Set[str] = set()
+        for g in groups:
+            g = [w for w in g if w]
+            if len(g) < 2:
+                continue
+            # longest common prefix and suffix over the group
+            pre = g[0]
+            suf = g[0]
+            for w in g[1:]:
+                while pre and not w.startswith(pre):
+                    pre = pre[:-1]
+                while suf and not w.endswith(suf):
+                    suf = suf[1:]
+            if len(pre) >= 2:
+                subs.add(pre)
+            if len(suf) >= 2:
+                subs.add(suf)
+        return subs
+
+    def _cost(self, counts: Counter, shared_subs: Set[str]) -> float:
+        usage: Counter = Counter()
+        for w, morphs in self.seg.items():
+            for m in morphs:
+                usage[m] += counts[w]
+        cost = _lex_cost(usage.keys()) + _corpus_cost(usage)
+        cost -= self.share_bonus * sum(1 for m in usage if m in shared_subs)
+        return cost
+
+    def _best_split(self, counts: Counter, shared_subs: Set[str]):
+        base = self._cost(counts, shared_subs)
+        best_gain, best = 1e-6, None
+        for w, morphs in self.seg.items():
+            for mi, m in enumerate(morphs):
+                if len(m) < 2 * _MIN_MORPH:
+                    continue
+                for cut in range(_MIN_MORPH, len(m) - _MIN_MORPH + 1):
+                    morphs[mi : mi + 1] = [m[:cut], m[cut:]]
+                    gain = base - self._cost(counts, shared_subs)
+                    morphs[mi : mi + 2] = [m]
+                    if gain > best_gain:
+                        best_gain, best = gain, (w, mi, cut)
+        return best
+
+    def segment(self, word: str) -> List[str]:
+        """Segment a word; unseen words are matched greedily against the
+        learned morph inventory (longest-match, both ends first)."""
+        if word in self.seg:
+            return list(self.seg[word])
+        morphs = {m for parts in self.seg.values() for m in parts}
+        return _greedy_decompose(word, morphs)
+
+    @property
+    def morphs(self) -> Set[str]:
+        return {m for parts in self.seg.values() for m in parts}
+
+
+def _greedy_decompose(word: str, morphs: Set[str]) -> List[str]:
+    """Best-effort decomposition of an unseen word over a morph set: dynamic
+    programming for fewest chunks, unknown spans kept as single chunks."""
+    n = len(word)
+    INF = float("inf")
+    # cost[i] = (num chunks, num unknown chars) to segment word[:i]
+    cost = [(INF, INF)] * (n + 1)
+    back: List[Optional[Tuple[int, str]]] = [None] * (n + 1)
+    cost[0] = (0, 0)
+    for i in range(n):
+        if cost[i][0] == INF:
+            continue
+        for j in range(i + 1, n + 1):
+            piece = word[i:j]
+            known = piece in morphs
+            c = (cost[i][0] + 1, cost[i][1] + (0 if known else len(piece)))
+            # prefer fewer unknown chars, then fewer chunks
+            key = (c[1], c[0])
+            if key < (cost[j][1], cost[j][0]):
+                cost[j] = c
+                back[j] = (i, piece)
+    out: List[str] = []
+    i = n
+    while i > 0 and back[i]:
+        prev, piece = back[i]
+        out.append(piece)
+        i = prev
+    out.reverse()
+    return out or [word]

solver/synth.py

@@ -0,0 +1,126 @@
+"""Program-synthesis subagent: LLM proposes grammars in the DSL, the
+interpreter executes them, the verifier scores them, and failing pairs are
+fed back for refinement (CEGIS), up to R rounds.
+
+The LLM never applies rules — it only emits grammar JSON. All execution is
+Interpreter; all selection is verifier.evaluate on the attested pairs.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Sequence, Tuple
+
+from .dsl.grammar import Grammar, from_json
+from .dsl.interpreter import Interpreter
+from .llm import LLMClient
+from .preprocess import Pair, Puzzle
+from .scaffold import analysis_blocks
+from .verifier import Verdict, evaluate
+
+PROMPT_DIR = Path(__file__).resolve().parents[1] / "prompts"
+MAX_FAILURES_SHOWN = 8
+
+
+@dataclass
+class SynthResult:
+    grammar: Optional[Grammar]
+    interpreter: Optional[Interpreter]
+    verdict: Optional[Verdict]
+    rounds_used: int = 0
+
+
+def _load_prompt(name: str) -> str:
+    return (PROMPT_DIR / f"{name}.md").read_text(encoding="utf-8")
+
+
+def proposer_prompt(puzzle: Puzzle) -> str:
+    seg_block, align_block = analysis_blocks(puzzle)
+    pairs_block = "\n".join(f"  {p.src}  =  {p.tgt}" for p in puzzle.pairs) or "  (none)"
+    hints_block = "\n".join(f"  {h}" for h in puzzle.hints) or "  (none)"
+    return _load_prompt("proposer").format(
+        task_lang=puzzle.task_lang or "the unknown language",
+        work_lang=puzzle.work_lang or "English",
+        pairs_block=pairs_block,
+        hints_block=hints_block,
+        segmentation_block=seg_block,
+        alignment_block=align_block,
+    )
+
+
+def refine_prompt(puzzle: Puzzle, grammar: Grammar, verdict: Verdict) -> str:
+    fails = verdict.failures[:MAX_FAILURES_SHOWN]
+    failures_block = "\n".join(
+        f"  input: {src}\n    expected: {gold}\n    got: {pred or '(nothing)'}"
+        for src, gold, pred in fails
+    )
+    return _load_prompt("refine").format(
+        task_lang=puzzle.task_lang or "the unknown language",
+        failures_block=failures_block,
+        previous_grammar=grammar.to_json(),
+    )
+
+
+def _attested_for_direction(pairs: Sequence[Pair], direction: str) -> List[Tuple[str, str]]:
+    if direction == "to_task":
+        return [(p.tgt, p.src) for p in pairs]  # work -> task (generation)
+    return [(p.src, p.tgt) for p in pairs]  # task -> work (analysis)
+
+
+def _predictor(interp: Interpreter, direction: str):
+    return interp.generate if direction == "to_task" else interp.analyze
+
+
+def score_grammar(g: Grammar, pairs: Sequence[Pair], direction: str) -> Tuple[Interpreter, Verdict]:
+    interp = Interpreter(g)
+    attested = _attested_for_direction(pairs, direction)
+    return interp, evaluate(_predictor(interp, direction), attested, g.mdl())
+
+
+def synthesize(
+    puzzle: Puzzle,
+    client: LLMClient,
+    direction: str = "to_task",
+    rounds: int = 2,
+) -> SynthResult:
+    """CEGIS loop: propose -> execute -> verify -> refine on failures.
+    Returns the best grammar seen across rounds (never a later-worse one)."""
+    if not client.available or not puzzle.pairs:
+        return SynthResult(None, None, None, 0)
+
+    best: SynthResult = SynthResult(None, None, None, 0)
+    prompt = proposer_prompt(puzzle)
+    for r in range(rounds + 1):
+        text = client.generate([prompt])[0]
+        g = from_json(text)
+        if g is None:
+            break
+        interp, verdict = score_grammar(g, puzzle.pairs, direction)
+        if best.verdict is None or verdict.score > best.verdict.score:
+            best = SynthResult(g, interp, verdict, r + 1)
+        if verdict.em >= 1.0 or r == rounds:
+            break
+        prompt = refine_prompt(puzzle, g, verdict)
+    return best
+
+
+def synthesize_best_of_n(
+    puzzle: Puzzle,
+    client: LLMClient,
+    direction: str,
+    n: int,
+    rounds: int = 1,
+) -> SynthResult:
+    """Phase-3 test-time scaling hook: N independent proposals (greedy base is
+    deterministic, so diversity must come from prompt variants), each with a
+    short CEGIS budget; verifier picks. With greedy decoding, n>1 only helps
+    once prompt variants or sampling adapters exist — the plumbing is here."""
+    best = SynthResult(None, None, None, 0)
+    for _ in range(max(1, n)):
+        r = synthesize(puzzle, client, direction, rounds)
+        if r.verdict and (best.verdict is None or r.verdict.score > best.verdict.score):
+            best = r
+        if best.verdict and best.verdict.em >= 1.0:
+            break
+    return best

solver/tables.py

@@ -0,0 +1,181 @@
+"""Table-completion solver: answer items that are rows of a paradigm table
+with one or more cells missing.
+
+Covers the recurring Linguini patterns:
+- fill_blanks with (k) markers in any column ("netkayʼ | (1) | push"),
+  including damaged rows where a marker merged with text ("*ʔikat | (4) | (5) to tie");
+- numbered query rows lacking one column the context table has
+  ("12. gsnqo'qon | foolishness" against context "word | [IPA] | gloss");
+- multi-language columns (Proto-Chamic | Phan Rang Cham | Tsat | meaning).
+
+Method per item:
+1. strip (k) markers; the remaining non-empty cell texts are the knowns;
+2. map knowns to context-table columns by character overlap (greedy);
+3. assign the row's markers, in order, to the free columns, preferring the
+   free column matching the marker's position in the row (first/last);
+4. build (known-column -> answer-column) pairs from the table and pick the
+   most learnable source column by leave-one-out template-translator fit;
+5. predict; abstention returns None (router falls through to LLM/fallback).
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Callable, List, Optional, Sequence, Tuple
+
+from .analogy import solve_from_pairs as analogy_vote
+from .fallback import fallback_answer
+from .preprocess import _BLANK_MARK, Pair, Puzzle, QueryItem
+from .template import TemplateTranslator
+from .verifier import leave_one_out
+
+
+def _col_overlap(cell: str, column: Sequence[str], n: int = 3) -> float:
+    s = "".join(cell.lower().split())
+    material = " ".join(c.lower() for c in column)
+    if len(s) < n:
+        return 1.0 if any(cell.strip() == c.strip() for c in column) else 0.0
+    grams = {s[i : i + n] for i in range(len(s) - n + 1)}
+    return sum(1 for g in grams if g in material) / max(len(grams), 1)
+
+
+def _clean_rows(table: List[List[str]]) -> List[List[str]]:
+    if not table:
+        return []
+    width = Counter(len(r) for r in table).most_common(1)[0][0]
+    return [r for r in table
+            if len(r) == width and not any(_BLANK_MARK.search(c) for c in r)]
+
+
+def pick_table(puzzle: Puzzle, min_width: int = 2) -> List[List[str]]:
+    best: List[List[str]] = []
+    for t in puzzle.tables:
+        rows = _clean_rows(t)
+        if rows and len(rows[0]) >= min_width and len(rows) > len(best):
+            best = rows
+    return best
+
+
+def _column_pairs(table: List[List[str]], src_col: int, ans_col: int) -> List[Pair]:
+    out = []
+    for r in table:
+        s, t = r[src_col].strip(), r[ans_col].strip()
+        if s and t and s != "-" and t != "-":
+            out.append(Pair(src=s, tgt=t, sep="table"))
+    return out
+
+
+def _predictor_for(table: List[List[str]], src_col: int, ans_col: int
+                   ) -> Tuple[float, Callable[[str], Optional[str]]]:
+    """LOO-scored predictor mapping src_col text to ans_col text. Chain:
+    template translation (multi-word rows), char-level analogy voting
+    (single-word paradigm columns: a:b :: query:x over all column pairs),
+    then the echo fallback."""
+    pairs = _column_pairs(table, src_col, ans_col)
+    if len(pairs) < 2:
+        return -1.0, lambda q: None
+    attested = [(p.src, p.tgt) for p in pairs]
+
+    def make(pair_list: List[Pair]) -> Callable[[str], Optional[str]]:
+        tmpl = TemplateTranslator(pair_list, "to_work")
+        ana = [(p.src, p.tgt) for p in pair_list]
+
+        def predict(q: str) -> Optional[str]:
+            ans = tmpl.translate(q)
+            if ans:
+                return ans
+            votes = analogy_vote(ana, q)
+            if votes:
+                return votes[0]
+            return fallback_answer(q, pair_list, "to_work")
+
+        return predict
+
+    def fit(held_in):
+        keep = set(held_in)
+        return make([p for p in pairs if (p.src, p.tgt) in keep])
+
+    v = leave_one_out(fit, attested)
+    return v.score, make(pairs)
+
+
+class TableSolver:
+    """Per-puzzle: caches the context table and column predictors."""
+
+    def __init__(self, puzzle: Puzzle):
+        self.table = pick_table(puzzle)
+        self._pred_cache: dict = {}
+
+    @property
+    def usable(self) -> bool:
+        return len(self.table) >= 2
+
+    def _predictor(self, src_col: int, ans_col: int):
+        key = (src_col, ans_col)
+        if key not in self._pred_cache:
+            self._pred_cache[key] = _predictor_for(self.table, src_col, ans_col)
+        return self._pred_cache[key]
+
+    def solve(self, item: QueryItem) -> Optional[Tuple[str, float]]:
+        """Returns (answer, confidence) or None. Confidence is the LOO fit of
+        the chosen column predictor."""
+        if not self.usable or not item.row:
+            return None
+        n_cols = len(self.table[0])
+        cols = [[r[c] for r in self.table] for c in range(n_cols)]
+
+        # knowns: marker-stripped non-empty cells, with their row position
+        knowns: List[Tuple[int, str]] = []
+        marker_pos: List[Tuple[int, str]] = []  # (cell index, marker number)
+        for ci, cell in enumerate(item.row):
+            for k in _BLANK_MARK.findall(cell):
+                marker_pos.append((ci, k))
+            text = _BLANK_MARK.sub("", cell).strip()
+            if text and text != "-":
+                knowns.append((ci, text))
+        if not marker_pos and item.blank_col is None:
+            return None
+
+        if len(item.row) == n_cols and item.blank_col is not None:
+            # same shape as the context table: mapping is positional and the
+            # marker's own column is the answer column
+            known_col = {ki: ci for ki, (ci, _) in enumerate(knowns)}
+            ans_col = item.blank_col
+        else:
+            # width mismatch (damaged/narrow rows): map knowns to table
+            # columns by char overlap with a positional prior as tie-break
+            scored = sorted(
+                ((_col_overlap(t, cols[c]) + 0.01 / (1 + abs(ci - c)), ki, c)
+                 for ki, (ci, t) in enumerate(knowns) for c in range(n_cols)),
+                reverse=True,
+            )
+            known_col = {}
+            used = set()
+            for s, ki, c in scored:
+                if ki in known_col or c in used:
+                    continue
+                known_col[ki] = c
+                used.add(c)
+            free = [c for c in range(n_cols) if c not in used]
+            if not free:
+                return None
+            # markers in row order take free columns in order; a single
+            # marker at the row's edge prefers the matching edge column
+            my_marker_idx = next(
+                (mi for mi, (_, k) in enumerate(marker_pos) if k == item.number), 0)
+            if len(marker_pos) <= 1 and item.row and len(free) > 1:
+                cell_idx = marker_pos[0][0] if marker_pos else (item.blank_col or 0)
+                ans_col = free[-1] if cell_idx >= len(item.row) - 1 else free[0]
+            else:
+                ans_col = free[min(my_marker_idx, len(free) - 1)]
+
+        # pick the most learnable known column as the source
+        best_score, best_predict, best_ki = -1.0, None, None
+        for ki, c in known_col.items():
+            score, predict = self._predictor(c, ans_col)
+            if score > best_score:
+                best_score, best_predict, best_ki = score, predict, ki
+        if best_predict is None or best_ki is None:
+            return None
+        ans = best_predict(knowns[best_ki][1])
+        return (ans, max(best_score, 0.0)) if ans else None

solver/template.py

@@ -0,0 +1,146 @@
+"""Template translation by minimal-pair substitution — the strongest
+zero-LLM translation baseline for constructed puzzles.
+
+Idea: puzzles are built so query sentences differ from attested ones by a
+small substitution. Find the attested pair whose source is closest to the
+query (token-level), then replace the differing tokens in its *target* using
+alignment links (align.py). Works in both directions. Also supports
+morph-level substitution for single-word queries (paradigm cells).
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Dict, List, Optional, Tuple
+
+from .align import align as build_align, one_to_one
+from .preprocess import Pair, strip_punct, tokenize
+
+
+def _toks(s: str) -> List[str]:
+    return [strip_punct(t).casefold() for t in tokenize(s) if strip_punct(t)]
+
+
+def _flip(pairs: List[Pair]) -> List[Pair]:
+    return [Pair(src=p.tgt, tgt=p.src) for p in pairs]
+
+
+class TemplateTranslator:
+    """direction 'to_work': translate task->work; 'to_task': work->task."""
+
+    def __init__(self, pairs: List[Pair], direction: str = "to_work"):
+        self.pairs = pairs if direction == "to_work" else _flip(pairs)
+        self.amap = build_align(self.pairs)  # src tok -> ranked [(tgt, score)]
+
+    def _sub(self, src_tok: str) -> Optional[str]:
+        cands = self.amap.get(src_tok.casefold())
+        return cands[0][0] if cands else None
+
+    def _sub_in(self, src_tok: str, pool: List[str]) -> Optional[str]:
+        """Best candidate for src_tok that is present in pool (context-aware:
+        a token may have both a bare and an inflected realization; the one
+        actually in the template target is the right one)."""
+        for c, _ in self.amap.get(src_tok.casefold(), []):
+            if c in pool:
+                return c
+        return None
+
+    def _sub_like(self, src_tok: str, model: str) -> Optional[str]:
+        """Best candidate for src_tok, preferring one that shares an affix
+        (prefix/suffix >= 2 chars) with `model` — the form it will replace.
+        kupu:nakupu :: moko:namoko."""
+        cands = self.amap.get(src_tok.casefold(), [])
+        for c, _ in cands:
+            if len(c) >= 2 and len(model) >= 2 and (c[:2] == model[:2] or c[-2:] == model[-2:]):
+                return c
+        return cands[0][0] if cands else None
+
+    def translate(self, query: str) -> Optional[str]:
+        q = _toks(query)
+        if not q:
+            return None
+        # rank templates by token-bag distance, then by length mismatch: a
+        # same-length template is a substitution frame; a much shorter one
+        # (e.g. a single-word gloss) would force fabricating structure
+        ranked = sorted(
+            ((_bag_distance(q, _toks(p.src)), abs(len(_toks(p.src)) - len(q)),
+              _toks(p.src), _toks(p.tgt)) for p in self.pairs),
+            key=lambda x: (x[0], x[1]),
+        )
+        max_dist = max(2, len(q) // 2)
+        for dist, _, s, t in ranked:
+            if dist == 0:
+                return " ".join(t)
+            if dist > max_dist:
+                break
+            out = self._substitute(q, s, t)
+            if out:
+                return out
+        return None
+
+    def _substitute(self, q: List[str], s: List[str], t: List[str]) -> Optional[str]:
+        """Swap the tokens where query and template source differ, mapping
+        both sides through the alignment. Abstains (None) when any needed
+        link is missing — a wrong-but-confident answer is worse than letting
+        the next template or the fallback ladder take over."""
+        q_extra = list((Counter(q) - Counter(s)).elements())
+        s_extra = list((Counter(s) - Counter(q)).elements())
+        out = list(t)
+        used: set = set()
+        for s_tok in s_extra:
+            s_tgt = self._sub_in(s_tok, out)
+            if s_tgt is None:
+                return None
+            repl = None
+            for qi, q_tok in enumerate(q_extra):
+                if qi in used:
+                    continue
+                q_tgt = self._sub_like(q_tok, s_tgt)
+                if q_tgt:
+                    repl = q_tgt
+                    used.add(qi)
+                    break
+            if repl is None:
+                return None
+            out[out.index(s_tgt)] = repl
+        for qi, q_tok in enumerate(q_extra):
+            if qi not in used:
+                q_tgt = self._sub(q_tok)
+                if q_tgt and q_tgt not in out:
+                    out.append(q_tgt)
+        return " ".join(out) if out else None
+
+
+def _bag_distance(a: List[str], b: List[str]) -> int:
+    ca, cb = Counter(a), Counter(b)
+    return sum((ca - cb).values()) + sum((cb - ca).values())
+
+
+def paradigm_complete(stem: str, pairs: List[Pair], cue: str = "") -> Optional[str]:
+    """Complete a paradigm cell: find attested form-pairs (a, b) sharing a
+    stem, group them by their string edit, and apply the dominant edit to
+    `stem`. `cue` (e.g. 'plural') restricts to pairs whose gloss relation
+    mentions the cue when glosses are available."""
+    from .analogy import edit_rules, apply_rule
+
+    vocab: Dict[str, str] = {}  # form -> gloss
+    for p in pairs:
+        if " " not in p.src.strip():
+            vocab[p.src.strip().casefold()] = p.tgt.strip().casefold()
+
+    rules: Counter = Counter()
+    for a in vocab:
+        for b in vocab:
+            if a != b and len(b) > len(a) and b.startswith(a[: max(2, len(a) - 1)]):
+                for r in edit_rules(a, b):
+                    if cue:
+                        ga, gb = vocab.get(a, ""), vocab.get(b, "")
+                        # cue must relate the two glosses (e.g. 'houses' vs 'house')
+                        if not (ga and gb and (ga in gb or gb in ga)):
+                            continue
+                    rules[r] += 1
+    for r, _ in rules.most_common(3):
+        out = apply_rule(r, stem.casefold())
+        if out and out != stem:
+            return out
+    return None

solver/verifier.py

@@ -0,0 +1,121 @@
+"""The verifier: leave-one-out fit of a candidate solver on attested pairs.
+
+One object, reused everywhere: grammar selection, CEGIS failure feedback, and
+(offline) RL reward. score = sqrt(EM * chrF) on held-out attested pairs,
+minus an MDL penalty so the simplest adequate grammar wins ties.
+
+A "candidate" is anything with `predict(src: str) -> str` for the relevant
+direction; grammars, analogy baselines, and raw LLM outputs all fit.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass, field
+from typing import Callable, List, Optional, Sequence, Tuple
+
+from .metrics import chrf, exact_match
+
+Predictor = Callable[[str], Optional[str]]
+
+
+@dataclass
+class Verdict:
+    em: float
+    chrf: float
+    mdl: float
+    failures: List[Tuple[str, str, str]] = field(default_factory=list)  # (src, gold, pred)
+
+    @property
+    def fit(self) -> float:
+        return math.sqrt(max(self.em, 0.0) * max(self.chrf, 0.0))
+
+    @property
+    def score(self) -> float:
+        return self.fit - self.mdl
+
+    def __repr__(self) -> str:
+        return f"Verdict(em={self.em:.3f}, chrf={self.chrf:.3f}, mdl={self.mdl:.4f}, n_fail={len(self.failures)})"
+
+
+def evaluate(
+    predict: Predictor,
+    pairs: Sequence[Tuple[str, str]],
+    mdl_cost: float = 0.0,
+    mdl_weight: float = 0.002,
+) -> Verdict:
+    """Score `predict` on attested (src, gold) pairs. A None/empty prediction
+    scores 0 on both metrics for that pair. mdl_cost is the grammar's
+    description length (see dsl.grammar.Grammar.mdl); weighted lightly so it
+    only breaks ties."""
+    if not pairs:
+        return Verdict(0.0, 0.0, mdl_cost * mdl_weight)
+    em_sum = chrf_sum = 0.0
+    failures = []
+    for src, gold in pairs:
+        pred = predict(src) or ""
+        e, c = exact_match(pred, gold), chrf(pred, gold)
+        em_sum += e
+        chrf_sum += c
+        if e < 1.0:
+            failures.append((src, gold, pred))
+    n = len(pairs)
+    return Verdict(em_sum / n, chrf_sum / n, mdl_cost * mdl_weight, failures)
+
+
+LOO_MAX_FOLDS = 12
+
+
+def leave_one_out(
+    fit_predict: Callable[[Sequence[Tuple[str, str]]], Predictor],
+    pairs: Sequence[Tuple[str, str]],
+    mdl_cost: float = 0.0,
+    max_folds: int = LOO_MAX_FOLDS,
+) -> Verdict:
+    """True LOO for candidates that are *fit* from pairs (analogy, alignment
+    baselines): refit without pair i, predict pair i. For a fixed grammar
+    (already synthesized), use `evaluate` directly — the LLM saw the pairs,
+    but the grammar either reproduces them or it doesn't.
+
+    Refitting is O(pairs^2)+ per fold (alignment rebuild), so folds are capped
+    at `max_folds` evenly-spaced held-out pairs — an unbiased estimate is all
+    the selection needs, and the 30-minute budget cannot afford exact LOO on
+    40-pair puzzles."""
+    if not pairs:
+        return Verdict(0.0, 0.0, 0.0)
+    n = len(pairs)
+    if n <= max_folds:
+        fold_idx = range(n)
+    else:
+        step = n / max_folds
+        fold_idx = sorted({int(k * step) for k in range(max_folds)})
+    em_sum = chrf_sum = 0.0
+    failures = []
+    for i in fold_idx:
+        src, gold = pairs[i]
+        held_in = [p for j, p in enumerate(pairs) if j != i]
+        pred = fit_predict(held_in)(src) or ""
+        e, c = exact_match(pred, gold), chrf(pred, gold)
+        em_sum += e
+        chrf_sum += c
+        if e < 1.0:
+            failures.append((src, gold, pred))
+    k = len(list(fold_idx))
+    return Verdict(em_sum / k, chrf_sum / k, mdl_cost * 0.002, failures)
+
+
+def select_best(
+    candidates: Sequence[Tuple[str, Predictor, float]],
+    pairs: Sequence[Tuple[str, str]],
+) -> Tuple[Optional[str], Optional[Predictor], Verdict]:
+    """Pick the best (name, predictor, mdl_cost) by verifier score.
+    Ties broken by lower MDL (already in score), then earlier order
+    (candidates should be ordered by prior preference: symbolic first)."""
+    best: Tuple[Optional[str], Optional[Predictor], Verdict] = (None, None, Verdict(0, 0, 0))
+    best_score = -1e9
+    for name, pred, mdl in candidates:
+        v = evaluate(pred, pairs, mdl)
+        if v.score > best_score + 1e-9:
+            best_score = v.score
+            best = (name, pred, v)
+    return best