293 lines
11 KiB
Python
293 lines
11 KiB
Python
"""Pure-Python harness logic for the IOL-AI 2026 submission.
|
||
|
||
Kept free of torch/transformers so it can be unit-tested locally without a GPU
|
||
or the model. ``script.py`` (which runs on the T4) imports from here; so does
|
||
``test_parsing.py``. Both this file and ``script.py`` are uploaded to the
|
||
submission repo, so ``import iol_harness`` resolves at run time (the repo root is
|
||
the working directory and on sys.path[0]).
|
||
"""
|
||
|
||
import ast
|
||
import json
|
||
import re
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Item counting
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
# "17. ", "18) ", ... anywhere after start/space/comma/semicolon, followed by
|
||
# whitespace or end-of-line. Requires whitespace after the . or ) so decimals
|
||
# like "3.14" are NOT matched.
|
||
_NUMBERED = re.compile(r"(?:^|[\s,;])(\d{1,3})[.)](?=\s|$)", re.MULTILINE)
|
||
# "(1)", "(2)" — parenthesised single number. "(1-2)" is NOT matched.
|
||
_PAREN = re.compile(r"\((\d{1,3})\)")
|
||
# Explicit item range in the query, e.g. "(1-9)", "(13–16)", "(1—10)".
|
||
_RANGE = re.compile(r"\((\d{1,3})\s*[-–—]\s*(\d{1,3})\)")
|
||
|
||
|
||
def _distinct(rx, text):
|
||
return sorted(set(int(x) for x in rx.findall(text or "")))
|
||
|
||
|
||
def _range_count(query):
|
||
"""Count from an explicit "(a-b)" range in the query, else 0."""
|
||
best = 0
|
||
for a, b in _RANGE.findall(query or ""):
|
||
a, b = int(a), int(b)
|
||
if b >= a:
|
||
best = max(best, b - a + 1)
|
||
return best
|
||
|
||
|
||
def _item_lines(query):
|
||
"""Non-empty lines after the first non-empty (instruction) line."""
|
||
lines = [ln for ln in (query or "").splitlines() if ln.strip()]
|
||
return max(0, len(lines) - 1)
|
||
|
||
|
||
def count_items(context, query, task_type=""):
|
||
"""How many answers the problem expects (validated on real Linguini data).
|
||
|
||
Items live in different places by task type: numbered/bare lines in the
|
||
query, blanks "(N)" or a numbered list in the context, or an explicit
|
||
"(a-b)" range in the query. Returns an int >= 1. Biased so that when in
|
||
doubt it does not under-count (the scorer ignores extra predictions but
|
||
zero-scores any item a short prediction fails to cover).
|
||
"""
|
||
tt = (task_type or "").strip()
|
||
|
||
# 1. An explicit range in the query is the strongest, cleanest signal.
|
||
r = _range_count(query)
|
||
if r:
|
||
return r
|
||
|
||
qn, qp = _distinct(_NUMBERED, query), _distinct(_PAREN, query)
|
||
cn, cp = _distinct(_NUMBERED, context), _distinct(_PAREN, context)
|
||
il = _item_lines(query)
|
||
|
||
if tt in ("text_to_num", "num_to_text"):
|
||
# numbers/words listed as bare lines after the instruction line
|
||
return max(il, len(qn), len(qp), 1)
|
||
if tt == "match_letters":
|
||
# items numbered in the query if present, else numbered in the context
|
||
return len(qn) or len(cn) or il or 1
|
||
if tt == "fill_blanks":
|
||
return max(len(qp), len(qn), len(cp), il, 1)
|
||
# translation / default
|
||
n = max(len(qn), len(qp))
|
||
if n <= 1:
|
||
# bare item lines in the query, else blanks enumerated in the context
|
||
n = il if il >= 1 else max(len(cp), 1)
|
||
return max(n, 1)
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Prompt building (task-type aware)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
SYSTEM_BASE = (
|
||
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
||
"problems. Every problem is fully self-contained: use ONLY the data, "
|
||
"examples and hints given to deduce the grammar, vocabulary and rules of "
|
||
"the language. Never rely on outside knowledge of the language — work the "
|
||
"pattern out from the given data. First reason it out silently, then answer "
|
||
"every numbered item.\n"
|
||
"OUTPUT FORMAT: respond with ONLY a JSON array of strings — one string per "
|
||
"numbered item, in the SAME order as the query. No keys, no numbering, no "
|
||
"commentary, nothing outside the array."
|
||
)
|
||
|
||
_TASK_GUIDANCE = {
|
||
"translation": (
|
||
"Each answer is the translation of that item, using the vocabulary and "
|
||
"grammar you deduced from the data. Answer in the language the query "
|
||
"names: 'into English' -> English; 'into <language>' -> that language. "
|
||
"Output only the translated text."
|
||
),
|
||
"fill_blanks": (
|
||
"Work out the rule from the complete rows, then fill each blank with the "
|
||
"missing form IN THE LANGUAGE BEING ANALYSED (the non-English / target "
|
||
"language) — never its English meaning. Output only that form."
|
||
),
|
||
"match_letters": (
|
||
"For each numbered item, output ONLY the single UPPERCASE LETTER "
|
||
"(A, B, C, ...) of its correct match. Never output the matched word, its "
|
||
"translation, or any text other than the letter."
|
||
),
|
||
"text_to_num": (
|
||
"Work out the number system from the examples, then COMPUTE each value "
|
||
'and write it in digits (e.g. "285"). Do NOT copy a number from the '
|
||
"examples — derive the value for each new item. Output only the digits."
|
||
),
|
||
"num_to_text": (
|
||
"Work out the number system from the examples, then CONSTRUCT each number "
|
||
"as words in the target (task) language using those rules. Do NOT copy an "
|
||
"example — build the form for the given number. Output only those words."
|
||
),
|
||
}
|
||
_TASK_GUIDANCE_DEFAULT = (
|
||
"Give exactly what each numbered item asks for, in the form the query "
|
||
"requests."
|
||
)
|
||
|
||
|
||
def task_guidance(task_type):
|
||
return _TASK_GUIDANCE.get((task_type or "").strip(), _TASK_GUIDANCE_DEFAULT)
|
||
|
||
|
||
SYSTEM_COT = (
|
||
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
||
"problems. Every problem is self-contained: use ONLY the given data to deduce "
|
||
"the language's rules — never outside knowledge of the language. Reason "
|
||
"step by step to work out the pattern, then give your answers.\n"
|
||
"FINAL LINE: after your reasoning, output ONLY a JSON array of strings — one "
|
||
"per numbered item, in order — as the very last thing in your reply."
|
||
)
|
||
|
||
|
||
def build_messages(context, query, task_type, n, cot=False):
|
||
"""Return chat messages. ``n`` = required answer count. ``cot`` = let the
|
||
model reason before emitting the final JSON array (for reasoning models)."""
|
||
base = SYSTEM_COT if cot else SYSTEM_BASE
|
||
system = (
|
||
f"{base}\n{task_guidance(task_type)}\n"
|
||
f"The JSON array must have EXACTLY {n} string{'s' if n != 1 else ''}."
|
||
)
|
||
tail = (
|
||
f"Answer all {n} item{'s' if n != 1 else ''} as a JSON array of "
|
||
f"{n} string{'s' if n != 1 else ''}, in order"
|
||
+ (", after your step-by-step reasoning." if cot else ".")
|
||
)
|
||
user = f"{(context or '').strip()}\n\n{(query or '').strip()}\n\n{tail}"
|
||
return [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": user},
|
||
]
|
||
|
||
|
||
def max_new_tokens_for(task_type, n, cot=False, cap=1024):
|
||
"""Heuristic generation budget. ``cot`` adds room for reasoning (bounded by
|
||
``cap`` to protect the 30-min limit on a T4)."""
|
||
per_item = 96 if (task_type or "") == "translation" else 40
|
||
answer_room = 160 + per_item * max(1, n)
|
||
if cot:
|
||
return int(min(cap, answer_room + 640))
|
||
return int(min(cap, answer_room))
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Answer parsing (robust, multi-layer)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
_FENCE = re.compile(r"```(?:json|python)?\s*(.*?)```", re.DOTALL | re.IGNORECASE)
|
||
_LINE_NUM = re.compile(r"^\s*[\(\[]?(\d{1,3})[\).\]:]\s*(.*\S)?\s*$")
|
||
_PREFIX = re.compile(r"^\s*(?:answers?|output|result)\s*[:\-]\s*", re.IGNORECASE)
|
||
|
||
|
||
def _to_str(x):
|
||
if x is None:
|
||
return ""
|
||
if isinstance(x, (list, tuple)):
|
||
# A nested item (e.g. multiple accepted forms) — join readably.
|
||
return " ".join(_to_str(e) for e in x)
|
||
return str(x)
|
||
|
||
|
||
def _strip_quotes(s):
|
||
s = s.strip()
|
||
if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`":
|
||
s = s[1:-1].strip()
|
||
return s
|
||
|
||
|
||
def _clean(s):
|
||
return _strip_quotes(_to_str(s).strip())
|
||
|
||
|
||
def _strip_fences(text):
|
||
m = _FENCE.search(text or "")
|
||
return m.group(1) if m else (text or "")
|
||
|
||
|
||
def _balanced_arrays(text):
|
||
"""Yield every top-level [...] substring (handles nesting)."""
|
||
depth, start = 0, -1
|
||
for i, ch in enumerate(text):
|
||
if ch == "[":
|
||
if depth == 0:
|
||
start = i
|
||
depth += 1
|
||
elif ch == "]" and depth > 0:
|
||
depth -= 1
|
||
if depth == 0 and start >= 0:
|
||
yield text[start : i + 1]
|
||
|
||
|
||
def extract_json_array(text):
|
||
"""Return the LAST bracketed list that parses, as a list of strings, else None.
|
||
|
||
'Last' matters for reasoning models: they emit <think>...</think> (often
|
||
containing brackets) and then the final answer array — we want that final one.
|
||
"""
|
||
if not text:
|
||
return None
|
||
for frag in reversed(list(_balanced_arrays(text))):
|
||
for parser in (json.loads, ast.literal_eval):
|
||
try:
|
||
value = parser(frag)
|
||
except Exception:
|
||
continue
|
||
if isinstance(value, list):
|
||
return [_clean(x) for x in value]
|
||
return None
|
||
|
||
|
||
def parse_numbered_lines(text):
|
||
"""Parse 'N. answer' / 'N) answer' / '(N) answer' lines -> ordered answers."""
|
||
found = {}
|
||
for line in (text or "").splitlines():
|
||
m = _LINE_NUM.match(line)
|
||
if m and m.group(2):
|
||
found[int(m.group(1))] = _clean(m.group(2))
|
||
if not found:
|
||
return []
|
||
return [found[k] for k in sorted(found)]
|
||
|
||
|
||
def _fit(values, n):
|
||
"""Force ``values`` to exactly ``n`` entries (truncate / pad with '')."""
|
||
values = list(values)[:n]
|
||
values += [""] * (n - len(values))
|
||
return values
|
||
|
||
|
||
def parse_answers(text, n):
|
||
"""Turn raw model output into exactly ``n`` cleaned answer strings.
|
||
|
||
Layers: JSON array -> numbered lines -> plain non-empty lines. Always
|
||
returns a list of length ``n``; extras are dropped, shortfalls padded so no
|
||
item is silently missing (the scorer aligns predictions by position).
|
||
"""
|
||
n = max(1, int(n))
|
||
inner = _strip_fences(text)
|
||
# Reasoning models wrap their scratch-work in <think>...</think>; the answer
|
||
# follows the closing tag. Keep only what comes after it.
|
||
if "</think>" in inner:
|
||
inner = inner.rsplit("</think>", 1)[1]
|
||
|
||
arr = extract_json_array(inner)
|
||
if arr:
|
||
return _fit(arr, n)
|
||
|
||
numbered = parse_numbered_lines(inner)
|
||
if numbered:
|
||
return _fit(numbered, n)
|
||
|
||
lines = [_clean(_PREFIX.sub("", ln)) for ln in inner.splitlines()]
|
||
lines = [ln for ln in lines if ln]
|
||
if lines:
|
||
return _fit(lines, n)
|
||
|
||
single = _clean(inner)
|
||
return _fit([single] if single else [], n)
|