qwen25-14b-awq-offline/iol_harness.py

"""Pure-Python harness logic for the IOL-AI 2026 submission.

Kept free of torch/transformers so it can be unit-tested locally without a GPU
or the model. ``script.py`` (which runs on the T4) imports from here; so does
``test_parsing.py``. Both this file and ``script.py`` are uploaded to the
submission repo, so ``import iol_harness`` resolves at run time (the repo root is
the working directory and on sys.path[0]).
"""

import ast
import json
import re

# --------------------------------------------------------------------------- #
# Item counting
# --------------------------------------------------------------------------- #

# "17. ", "18) ", ... anywhere after start/space/comma/semicolon, followed by
# whitespace or end-of-line. Requires whitespace after the . or ) so decimals
# like "3.14" are NOT matched.
_NUMBERED = re.compile(r"(?:^|[\s,;])(\d{1,3})[.)](?=\s|$)", re.MULTILINE)
# "(1)", "(2)" — parenthesised single number. "(1-2)" is NOT matched.
_PAREN = re.compile(r"\((\d{1,3})\)")
# Explicit item range in the query, e.g. "(1-9)", "(13–16)", "(1—10)".
_RANGE = re.compile(r"\((\d{1,3})\s*[-–—]\s*(\d{1,3})\)")


def _distinct(rx, text):
    return sorted(set(int(x) for x in rx.findall(text or "")))


def _range_count(query):
    """Count from an explicit "(a-b)" range in the query, else 0."""
    best = 0
    for a, b in _RANGE.findall(query or ""):
        a, b = int(a), int(b)
        if b >= a:
            best = max(best, b - a + 1)
    return best


def _item_lines(query):
    """Non-empty lines after the first non-empty (instruction) line."""
    lines = [ln for ln in (query or "").splitlines() if ln.strip()]
    return max(0, len(lines) - 1)


def count_items(context, query, task_type=""):
    """How many answers the problem expects (validated on real Linguini data).

    Items live in different places by task type: numbered/bare lines in the
    query, blanks "(N)" or a numbered list in the context, or an explicit
    "(a-b)" range in the query. Returns an int >= 1. Biased so that when in
    doubt it does not under-count (the scorer ignores extra predictions but
    zero-scores any item a short prediction fails to cover).
    """
    tt = (task_type or "").strip()

    # 1. An explicit range in the query is the strongest, cleanest signal.
    r = _range_count(query)
    if r:
        return r

    qn, qp = _distinct(_NUMBERED, query), _distinct(_PAREN, query)
    cn, cp = _distinct(_NUMBERED, context), _distinct(_PAREN, context)
    il = _item_lines(query)

    if tt in ("text_to_num", "num_to_text"):
        # numbers/words listed as bare lines after the instruction line
        return max(il, len(qn), len(qp), 1)
    if tt == "match_letters":
        # items numbered in the query if present, else numbered in the context
        return len(qn) or len(cn) or il or 1
    if tt == "fill_blanks":
        return max(len(qp), len(qn), len(cp), il, 1)
    # translation / default
    n = max(len(qn), len(qp))
    if n <= 1:
        # bare item lines in the query, else blanks enumerated in the context
        n = il if il >= 1 else max(len(cp), 1)
    return max(n, 1)


# --------------------------------------------------------------------------- #
# Prompt building (task-type aware)
# --------------------------------------------------------------------------- #

SYSTEM_BASE = (
    "You are an expert solver of International Linguistics Olympiad (IOL) "
    "problems. Every problem is fully self-contained: use ONLY the data, "
    "examples and hints given to deduce the grammar, vocabulary and rules of "
    "the language. Never rely on outside knowledge of the language — work the "
    "pattern out from the given data. First reason it out silently, then answer "
    "every numbered item.\n"
    "OUTPUT FORMAT: respond with ONLY a JSON array of strings — one string per "
    "numbered item, in the SAME order as the query. No keys, no numbering, no "
    "commentary, nothing outside the array."
)

_TASK_GUIDANCE = {
    "translation": (
        "Each answer is the translation of that item, using the vocabulary and "
        "grammar you deduced from the data. Answer in the language the query "
        "names: 'into English' -> English; 'into <language>' -> that language. "
        "Output only the translated text."
    ),
    "fill_blanks": (
        "Work out the rule from the complete rows, then fill each blank with the "
        "missing form IN THE LANGUAGE BEING ANALYSED (the non-English / target "
        "language) — never its English meaning. Output only that form."
    ),
    "match_letters": (
        "For each numbered item, output ONLY the single UPPERCASE LETTER "
        "(A, B, C, ...) of its correct match. Never output the matched word, its "
        "translation, or any text other than the letter."
    ),
    "text_to_num": (
        "Work out the number system from the examples, then COMPUTE each value "
        'and write it in digits (e.g. "285"). Do NOT copy a number from the '
        "examples — derive the value for each new item. Output only the digits."
    ),
    "num_to_text": (
        "Work out the number system from the examples, then CONSTRUCT each number "
        "as words in the target (task) language using those rules. Do NOT copy an "
        "example — build the form for the given number. Output only those words."
    ),
}
_TASK_GUIDANCE_DEFAULT = (
    "Give exactly what each numbered item asks for, in the form the query "
    "requests."
)


def task_guidance(task_type):
    return _TASK_GUIDANCE.get((task_type or "").strip(), _TASK_GUIDANCE_DEFAULT)


SYSTEM_COT = (
    "You are an expert solver of International Linguistics Olympiad (IOL) "
    "problems. Every problem is self-contained: use ONLY the given data to deduce "
    "the language's rules — never outside knowledge of the language. Reason "
    "step by step to work out the pattern, then give your answers.\n"
    "FINAL LINE: after your reasoning, output ONLY a JSON array of strings — one "
    "per numbered item, in order — as the very last thing in your reply."
)


# One compact, fully synthetic worked example per task type. These teach the
# deduce-then-answer pattern and the terse JSON output form WITHOUT using any
# real IOL/Linguini data (no contamination). Each answer is verified correct.
FEWSHOT = {
    "translation": [{
        "user": ("Data from the language Nuu:\nka mi = I see\nka tu = you see\n"
                 "lo mi = I go\nka mi ne = I saw\n\nTranslate into English:\n"
                 "1. lo tu\n2. ka tu ne"),
        "assistant": '["you go", "you saw"]'}],
    "text_to_num": [{
        "user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before "
                 "ones: 'ka ta' = 11.\n\nWrite in digits:\n1. ka ba\n2. ta"),
        "assistant": '["12", "1"]'}],
    "num_to_text": [{
        "user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before "
                 "ones: 'ka ta' = 11.\n\nWrite in Zaz:\n1. 12\n2. 10"),
        "assistant": '["ka ba", "ka"]'}],
    "fill_blanks": [{
        "user": ("Verb forms:\nsing | singem | to sing\ndance | (1) | to dance\n\n"
                 "Fill the blanks (1)."),
        "assistant": '["dancem"]'}],
    "match_letters": [{
        "user": ("Clues: 'bo' occurs with fire, 'ka' with stone, 'mi' with water.\n"
                 "Words:\n1. mi\n2. bo\n3. ka\nMeanings:\nA. stone\nB. water\nC. fire\n\n"
                 "Determine the correct correspondences."),
        "assistant": '["B", "C", "A"]'}],
}


def build_messages(context, query, task_type, n, cot=False, fewshot=False):
    """Return chat messages. ``n`` = required answer count. ``cot`` = let the
    model reason before the final JSON array. ``fewshot`` = prepend a synthetic
    worked example for this task type (demonstrates the deduce-then-answer
    pattern and terse output form)."""
    base = SYSTEM_COT if cot else SYSTEM_BASE
    system = (
        f"{base}\n{task_guidance(task_type)}\n"
        f"The JSON array must have EXACTLY {n} string{'s' if n != 1 else ''}."
    )
    tail = (
        f"Answer all {n} item{'s' if n != 1 else ''} as a JSON array of "
        f"{n} string{'s' if n != 1 else ''}, in order"
        + (", after your step-by-step reasoning." if cot else ".")
    )
    user = f"{(context or '').strip()}\n\n{(query or '').strip()}\n\n{tail}"

    msgs = [{"role": "system", "content": system}]
    if fewshot:
        for ex in FEWSHOT.get((task_type or "").strip(), []):
            msgs.append({"role": "user", "content": ex["user"]})
            msgs.append({"role": "assistant", "content": ex["assistant"]})
    msgs.append({"role": "user", "content": user})
    return msgs


def max_new_tokens_for(task_type, n, cot=False, cap=1024):
    """Heuristic generation budget. ``cot`` adds room for reasoning (bounded by
    ``cap`` to protect the 30-min limit on a T4)."""
    per_item = 96 if (task_type or "") == "translation" else 40
    answer_room = 160 + per_item * max(1, n)
    if cot:
        return int(min(cap, answer_room + 640))
    return int(min(cap, answer_room))


# --------------------------------------------------------------------------- #
# Answer parsing (robust, multi-layer)
# --------------------------------------------------------------------------- #

_FENCE = re.compile(r"```(?:json|python)?\s*(.*?)```", re.DOTALL | re.IGNORECASE)
_LINE_NUM = re.compile(r"^\s*[\(\[]?(\d{1,3})[\).\]:]\s*(.*\S)?\s*$")
_PREFIX = re.compile(r"^\s*(?:answers?|output|result)\s*[:\-]\s*", re.IGNORECASE)


def _to_str(x):
    if x is None:
        return ""
    if isinstance(x, (list, tuple)):
        # A nested item (e.g. multiple accepted forms) — join readably.
        return " ".join(_to_str(e) for e in x)
    return str(x)


def _strip_quotes(s):
    s = s.strip()
    if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`":
        s = s[1:-1].strip()
    return s


def _clean(s):
    return _strip_quotes(_to_str(s).strip())


def _strip_fences(text):
    m = _FENCE.search(text or "")
    return m.group(1) if m else (text or "")


def _balanced_arrays(text):
    """Yield every top-level [...] substring (handles nesting)."""
    depth, start = 0, -1
    for i, ch in enumerate(text):
        if ch == "[":
            if depth == 0:
                start = i
            depth += 1
        elif ch == "]" and depth > 0:
            depth -= 1
            if depth == 0 and start >= 0:
                yield text[start : i + 1]


def extract_json_array(text):
    """Return the LAST bracketed list that parses, as a list of strings, else None.

    'Last' matters for reasoning models: they emit <think>...</think> (often
    containing brackets) and then the final answer array — we want that final one.
    """
    if not text:
        return None
    for frag in reversed(list(_balanced_arrays(text))):
        for parser in (json.loads, ast.literal_eval):
            try:
                value = parser(frag)
            except Exception:
                continue
            if isinstance(value, list):
                return [_clean(x) for x in value]
    return None


def parse_numbered_lines(text):
    """Parse 'N. answer' / 'N) answer' / '(N) answer' lines -> ordered answers."""
    found = {}
    for line in (text or "").splitlines():
        m = _LINE_NUM.match(line)
        if m and m.group(2):
            found[int(m.group(1))] = _clean(m.group(2))
    if not found:
        return []
    return [found[k] for k in sorted(found)]


def _fit(values, n):
    """Force ``values`` to exactly ``n`` entries (truncate / pad with '')."""
    values = list(values)[:n]
    values += [""] * (n - len(values))
    return values


def parse_answers(text, n):
    """Turn raw model output into exactly ``n`` cleaned answer strings.

    Layers: JSON array -> numbered lines -> plain non-empty lines. Always
    returns a list of length ``n``; extras are dropped, shortfalls padded so no
    item is silently missing (the scorer aligns predictions by position).
    """
    n = max(1, int(n))
    inner = _strip_fences(text)
    # Reasoning models wrap their scratch-work in <think>...</think>; the answer
    # follows the closing tag. Keep only what comes after it.
    if "</think>" in inner:
        inner = inner.rsplit("</think>", 1)[1]

    arr = extract_json_array(inner)
    if arr:
        return _fit(arr, n)

    numbered = parse_numbered_lines(inner)
    if numbered:
        return _fit(numbered, n)

    lines = [_clean(_PREFIX.sub("", ln)) for ln in inner.splitlines()]
    lines = [ln for ln in lines if ln]
    if lines:
        return _fit(lines, n)

    single = _clean(inner)
    return _fit([single] if single else [], n)


# --------------------------------------------------------------------------- #
# Self-consistency: majority vote across passes
# --------------------------------------------------------------------------- #

def majority_vote(passes, n):
    """Per-item majority vote across ``passes`` (each a length-n answer list).

    Votes are counted case/space-insensitively but the winner keeps its original
    casing. Ties break toward the EARLIEST pass — so pass 0 (greedy) acts as a
    floor: confident items keep the greedy answer, uncertain ones adopt the
    consensus. Non-empty answers are preferred over blanks. Always returns n items.
    """
    n = max(1, int(n))
    out = []
    for i in range(n):
        col = [p[i] for p in passes if i < len(p)]
        counts, first = {}, {}
        for k, ans in enumerate(col):
            key = _to_str(ans).strip().lower()
            counts[key] = counts.get(key, 0) + 1
            if key not in first:
                first[key] = (k, ans)
        # drop the empty-string option unless it is all we have
        nonblank = {k_: v for k_, v in counts.items() if k_ != ""}
        pool = nonblank or counts
        if not pool:
            out.append("")
            continue
        best = max(pool, key=lambda kk: (pool[kk], -first[kk][0]))
        out.append(first[best][1])
    return out