293 lines
11 KiB
Python
293 lines
11 KiB
Python
|
"""Pure-Python harness logic for the IOL-AI 2026 submission.
|
|||
|
|
|
|||
|
|
Kept free of torch/transformers so it can be unit-tested locally without a GPU
|
|||
|
|
or the model. ``script.py`` (which runs on the T4) imports from here; so does
|
|||
|
|
``test_parsing.py``. Both this file and ``script.py`` are uploaded to the
|
|||
|
|
submission repo, so ``import iol_harness`` resolves at run time (the repo root is
|
|||
|
|
the working directory and on sys.path[0]).
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import ast
|
|||
|
|
import json
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
# Item counting
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
|
|||
|
|
# "17. ", "18) ", ... anywhere after start/space/comma/semicolon, followed by
|
|||
|
|
# whitespace or end-of-line. Requires whitespace after the . or ) so decimals
|
|||
|
|
# like "3.14" are NOT matched.
|
|||
|
|
_NUMBERED = re.compile(r"(?:^|[\s,;])(\d{1,3})[.)](?=\s|$)", re.MULTILINE)
|
|||
|
|
# "(1)", "(2)" — parenthesised single number. "(1-2)" is NOT matched.
|
|||
|
|
_PAREN = re.compile(r"\((\d{1,3})\)")
|
|||
|
|
# Explicit item range in the query, e.g. "(1-9)", "(13–16)", "(1—10)".
|
|||
|
|
_RANGE = re.compile(r"\((\d{1,3})\s*[-–—]\s*(\d{1,3})\)")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _distinct(rx, text):
|
|||
|
|
return sorted(set(int(x) for x in rx.findall(text or "")))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _range_count(query):
|
|||
|
|
"""Count from an explicit "(a-b)" range in the query, else 0."""
|
|||
|
|
best = 0
|
|||
|
|
for a, b in _RANGE.findall(query or ""):
|
|||
|
|
a, b = int(a), int(b)
|
|||
|
|
if b >= a:
|
|||
|
|
best = max(best, b - a + 1)
|
|||
|
|
return best
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _item_lines(query):
|
|||
|
|
"""Non-empty lines after the first non-empty (instruction) line."""
|
|||
|
|
lines = [ln for ln in (query or "").splitlines() if ln.strip()]
|
|||
|
|
return max(0, len(lines) - 1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def count_items(context, query, task_type=""):
|
|||
|
|
"""How many answers the problem expects (validated on real Linguini data).
|
|||
|
|
|
|||
|
|
Items live in different places by task type: numbered/bare lines in the
|
|||
|
|
query, blanks "(N)" or a numbered list in the context, or an explicit
|
|||
|
|
"(a-b)" range in the query. Returns an int >= 1. Biased so that when in
|
|||
|
|
doubt it does not under-count (the scorer ignores extra predictions but
|
|||
|
|
zero-scores any item a short prediction fails to cover).
|
|||
|
|
"""
|
|||
|
|
tt = (task_type or "").strip()
|
|||
|
|
|
|||
|
|
# 1. An explicit range in the query is the strongest, cleanest signal.
|
|||
|
|
r = _range_count(query)
|
|||
|
|
if r:
|
|||
|
|
return r
|
|||
|
|
|
|||
|
|
qn, qp = _distinct(_NUMBERED, query), _distinct(_PAREN, query)
|
|||
|
|
cn, cp = _distinct(_NUMBERED, context), _distinct(_PAREN, context)
|
|||
|
|
il = _item_lines(query)
|
|||
|
|
|
|||
|
|
if tt in ("text_to_num", "num_to_text"):
|
|||
|
|
# numbers/words listed as bare lines after the instruction line
|
|||
|
|
return max(il, len(qn), len(qp), 1)
|
|||
|
|
if tt == "match_letters":
|
|||
|
|
# items numbered in the query if present, else numbered in the context
|
|||
|
|
return len(qn) or len(cn) or il or 1
|
|||
|
|
if tt == "fill_blanks":
|
|||
|
|
return max(len(qp), len(qn), len(cp), il, 1)
|
|||
|
|
# translation / default
|
|||
|
|
n = max(len(qn), len(qp))
|
|||
|
|
if n <= 1:
|
|||
|
|
# bare item lines in the query, else blanks enumerated in the context
|
|||
|
|
n = il if il >= 1 else max(len(cp), 1)
|
|||
|
|
return max(n, 1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
# Prompt building (task-type aware)
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
|
|||
|
|
SYSTEM_BASE = (
|
|||
|
|
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
|||
|
|
"problems. Every problem is fully self-contained: use ONLY the data, "
|
|||
|
|
"examples and hints given to deduce the grammar, vocabulary and rules of "
|
|||
|
|
"the language. Never rely on outside knowledge of the language — work the "
|
|||
|
|
"pattern out from the given data. First reason it out silently, then answer "
|
|||
|
|
"every numbered item.\n"
|
|||
|
|
"OUTPUT FORMAT: respond with ONLY a JSON array of strings — one string per "
|
|||
|
|
"numbered item, in the SAME order as the query. No keys, no numbering, no "
|
|||
|
|
"commentary, nothing outside the array."
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
_TASK_GUIDANCE = {
|
|||
|
|
"translation": (
|
|||
|
|
"Each answer is the translation of that item, using the vocabulary and "
|
|||
|
|
"grammar you deduced from the data. Answer in the language the query "
|
|||
|
|
"names: 'into English' -> English; 'into <language>' -> that language. "
|
|||
|
|
"Output only the translated text."
|
|||
|
|
),
|
|||
|
|
"fill_blanks": (
|
|||
|
|
"Work out the rule from the complete rows, then fill each blank with the "
|
|||
|
|
"missing form IN THE LANGUAGE BEING ANALYSED (the non-English / target "
|
|||
|
|
"language) — never its English meaning. Output only that form."
|
|||
|
|
),
|
|||
|
|
"match_letters": (
|
|||
|
|
"For each numbered item, output ONLY the single UPPERCASE LETTER "
|
|||
|
|
"(A, B, C, ...) of its correct match. Never output the matched word, its "
|
|||
|
|
"translation, or any text other than the letter."
|
|||
|
|
),
|
|||
|
|
"text_to_num": (
|
|||
|
|
"Work out the number system from the examples, then COMPUTE each value "
|
|||
|
|
'and write it in digits (e.g. "285"). Do NOT copy a number from the '
|
|||
|
|
"examples — derive the value for each new item. Output only the digits."
|
|||
|
|
),
|
|||
|
|
"num_to_text": (
|
|||
|
|
"Work out the number system from the examples, then CONSTRUCT each number "
|
|||
|
|
"as words in the target (task) language using those rules. Do NOT copy an "
|
|||
|
|
"example — build the form for the given number. Output only those words."
|
|||
|
|
),
|
|||
|
|
}
|
|||
|
|
_TASK_GUIDANCE_DEFAULT = (
|
|||
|
|
"Give exactly what each numbered item asks for, in the form the query "
|
|||
|
|
"requests."
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def task_guidance(task_type):
|
|||
|
|
return _TASK_GUIDANCE.get((task_type or "").strip(), _TASK_GUIDANCE_DEFAULT)
|
|||
|
|
|
|||
|
|
|
|||
|
|
SYSTEM_COT = (
|
|||
|
|
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
|||
|
|
"problems. Every problem is self-contained: use ONLY the given data to deduce "
|
|||
|
|
"the language's rules — never outside knowledge of the language. Reason "
|
|||
|
|
"step by step to work out the pattern, then give your answers.\n"
|
|||
|
|
"FINAL LINE: after your reasoning, output ONLY a JSON array of strings — one "
|
|||
|
|
"per numbered item, in order — as the very last thing in your reply."
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_messages(context, query, task_type, n, cot=False):
|
|||
|
|
"""Return chat messages. ``n`` = required answer count. ``cot`` = let the
|
|||
|
|
model reason before emitting the final JSON array (for reasoning models)."""
|
|||
|
|
base = SYSTEM_COT if cot else SYSTEM_BASE
|
|||
|
|
system = (
|
|||
|
|
f"{base}\n{task_guidance(task_type)}\n"
|
|||
|
|
f"The JSON array must have EXACTLY {n} string{'s' if n != 1 else ''}."
|
|||
|
|
)
|
|||
|
|
tail = (
|
|||
|
|
f"Answer all {n} item{'s' if n != 1 else ''} as a JSON array of "
|
|||
|
|
f"{n} string{'s' if n != 1 else ''}, in order"
|
|||
|
|
+ (", after your step-by-step reasoning." if cot else ".")
|
|||
|
|
)
|
|||
|
|
user = f"{(context or '').strip()}\n\n{(query or '').strip()}\n\n{tail}"
|
|||
|
|
return [
|
|||
|
|
{"role": "system", "content": system},
|
|||
|
|
{"role": "user", "content": user},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def max_new_tokens_for(task_type, n, cot=False, cap=1024):
|
|||
|
|
"""Heuristic generation budget. ``cot`` adds room for reasoning (bounded by
|
|||
|
|
``cap`` to protect the 30-min limit on a T4)."""
|
|||
|
|
per_item = 96 if (task_type or "") == "translation" else 40
|
|||
|
|
answer_room = 160 + per_item * max(1, n)
|
|||
|
|
if cot:
|
|||
|
|
return int(min(cap, answer_room + 640))
|
|||
|
|
return int(min(cap, answer_room))
|
|||
|
|
|
|||
|
|
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
# Answer parsing (robust, multi-layer)
|
|||
|
|
# --------------------------------------------------------------------------- #
|
|||
|
|
|
|||
|
|
_FENCE = re.compile(r"```(?:json|python)?\s*(.*?)```", re.DOTALL | re.IGNORECASE)
|
|||
|
|
_LINE_NUM = re.compile(r"^\s*[\(\[]?(\d{1,3})[\).\]:]\s*(.*\S)?\s*$")
|
|||
|
|
_PREFIX = re.compile(r"^\s*(?:answers?|output|result)\s*[:\-]\s*", re.IGNORECASE)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _to_str(x):
|
|||
|
|
if x is None:
|
|||
|
|
return ""
|
|||
|
|
if isinstance(x, (list, tuple)):
|
|||
|
|
# A nested item (e.g. multiple accepted forms) — join readably.
|
|||
|
|
return " ".join(_to_str(e) for e in x)
|
|||
|
|
return str(x)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _strip_quotes(s):
|
|||
|
|
s = s.strip()
|
|||
|
|
if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`":
|
|||
|
|
s = s[1:-1].strip()
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _clean(s):
|
|||
|
|
return _strip_quotes(_to_str(s).strip())
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _strip_fences(text):
|
|||
|
|
m = _FENCE.search(text or "")
|
|||
|
|
return m.group(1) if m else (text or "")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _balanced_arrays(text):
|
|||
|
|
"""Yield every top-level [...] substring (handles nesting)."""
|
|||
|
|
depth, start = 0, -1
|
|||
|
|
for i, ch in enumerate(text):
|
|||
|
|
if ch == "[":
|
|||
|
|
if depth == 0:
|
|||
|
|
start = i
|
|||
|
|
depth += 1
|
|||
|
|
elif ch == "]" and depth > 0:
|
|||
|
|
depth -= 1
|
|||
|
|
if depth == 0 and start >= 0:
|
|||
|
|
yield text[start : i + 1]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_json_array(text):
|
|||
|
|
"""Return the LAST bracketed list that parses, as a list of strings, else None.
|
|||
|
|
|
|||
|
|
'Last' matters for reasoning models: they emit <think>...</think> (often
|
|||
|
|
containing brackets) and then the final answer array — we want that final one.
|
|||
|
|
"""
|
|||
|
|
if not text:
|
|||
|
|
return None
|
|||
|
|
for frag in reversed(list(_balanced_arrays(text))):
|
|||
|
|
for parser in (json.loads, ast.literal_eval):
|
|||
|
|
try:
|
|||
|
|
value = parser(frag)
|
|||
|
|
except Exception:
|
|||
|
|
continue
|
|||
|
|
if isinstance(value, list):
|
|||
|
|
return [_clean(x) for x in value]
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_numbered_lines(text):
|
|||
|
|
"""Parse 'N. answer' / 'N) answer' / '(N) answer' lines -> ordered answers."""
|
|||
|
|
found = {}
|
|||
|
|
for line in (text or "").splitlines():
|
|||
|
|
m = _LINE_NUM.match(line)
|
|||
|
|
if m and m.group(2):
|
|||
|
|
found[int(m.group(1))] = _clean(m.group(2))
|
|||
|
|
if not found:
|
|||
|
|
return []
|
|||
|
|
return [found[k] for k in sorted(found)]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fit(values, n):
|
|||
|
|
"""Force ``values`` to exactly ``n`` entries (truncate / pad with '')."""
|
|||
|
|
values = list(values)[:n]
|
|||
|
|
values += [""] * (n - len(values))
|
|||
|
|
return values
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse_answers(text, n):
|
|||
|
|
"""Turn raw model output into exactly ``n`` cleaned answer strings.
|
|||
|
|
|
|||
|
|
Layers: JSON array -> numbered lines -> plain non-empty lines. Always
|
|||
|
|
returns a list of length ``n``; extras are dropped, shortfalls padded so no
|
|||
|
|
item is silently missing (the scorer aligns predictions by position).
|
|||
|
|
"""
|
|||
|
|
n = max(1, int(n))
|
|||
|
|
inner = _strip_fences(text)
|
|||
|
|
# Reasoning models wrap their scratch-work in <think>...</think>; the answer
|
|||
|
|
# follows the closing tag. Keep only what comes after it.
|
|||
|
|
if "</think>" in inner:
|
|||
|
|
inner = inner.rsplit("</think>", 1)[1]
|
|||
|
|
|
|||
|
|
arr = extract_json_array(inner)
|
|||
|
|
if arr:
|
|||
|
|
return _fit(arr, n)
|
|||
|
|
|
|||
|
|
numbered = parse_numbered_lines(inner)
|
|||
|
|
if numbered:
|
|||
|
|
return _fit(numbered, n)
|
|||
|
|
|
|||
|
|
lines = [_clean(_PREFIX.sub("", ln)) for ln in inner.splitlines()]
|
|||
|
|
lines = [ln for ln in lines if ln]
|
|||
|
|
if lines:
|
|||
|
|
return _fit(lines, n)
|
|||
|
|
|
|||
|
|
single = _clean(inner)
|
|||
|
|
return _fit([single] if single else [], n)
|