361 lines
14 KiB
Python
361 lines
14 KiB
Python
"""Pure-Python harness logic for the IOL-AI 2026 submission.
|
||
|
||
Kept free of torch/transformers so it can be unit-tested locally without a GPU
|
||
or the model. ``script.py`` (which runs on the T4) imports from here; so does
|
||
``test_parsing.py``. Both this file and ``script.py`` are uploaded to the
|
||
submission repo, so ``import iol_harness`` resolves at run time (the repo root is
|
||
the working directory and on sys.path[0]).
|
||
"""
|
||
|
||
import ast
|
||
import json
|
||
import re
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Item counting
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
# "17. ", "18) ", ... anywhere after start/space/comma/semicolon, followed by
|
||
# whitespace or end-of-line. Requires whitespace after the . or ) so decimals
|
||
# like "3.14" are NOT matched.
|
||
_NUMBERED = re.compile(r"(?:^|[\s,;])(\d{1,3})[.)](?=\s|$)", re.MULTILINE)
|
||
# "(1)", "(2)" — parenthesised single number. "(1-2)" is NOT matched.
|
||
_PAREN = re.compile(r"\((\d{1,3})\)")
|
||
# Explicit item range in the query, e.g. "(1-9)", "(13–16)", "(1—10)".
|
||
_RANGE = re.compile(r"\((\d{1,3})\s*[-–—]\s*(\d{1,3})\)")
|
||
|
||
|
||
def _distinct(rx, text):
|
||
return sorted(set(int(x) for x in rx.findall(text or "")))
|
||
|
||
|
||
def _range_count(query):
|
||
"""Count from an explicit "(a-b)" range in the query, else 0."""
|
||
best = 0
|
||
for a, b in _RANGE.findall(query or ""):
|
||
a, b = int(a), int(b)
|
||
if b >= a:
|
||
best = max(best, b - a + 1)
|
||
return best
|
||
|
||
|
||
def _item_lines(query):
|
||
"""Non-empty lines after the first non-empty (instruction) line."""
|
||
lines = [ln for ln in (query or "").splitlines() if ln.strip()]
|
||
return max(0, len(lines) - 1)
|
||
|
||
|
||
def count_items(context, query, task_type=""):
|
||
"""How many answers the problem expects (validated on real Linguini data).
|
||
|
||
Items live in different places by task type: numbered/bare lines in the
|
||
query, blanks "(N)" or a numbered list in the context, or an explicit
|
||
"(a-b)" range in the query. Returns an int >= 1. Biased so that when in
|
||
doubt it does not under-count (the scorer ignores extra predictions but
|
||
zero-scores any item a short prediction fails to cover).
|
||
"""
|
||
tt = (task_type or "").strip()
|
||
|
||
# 1. An explicit range in the query is the strongest, cleanest signal.
|
||
r = _range_count(query)
|
||
if r:
|
||
return r
|
||
|
||
qn, qp = _distinct(_NUMBERED, query), _distinct(_PAREN, query)
|
||
cn, cp = _distinct(_NUMBERED, context), _distinct(_PAREN, context)
|
||
il = _item_lines(query)
|
||
|
||
if tt in ("text_to_num", "num_to_text"):
|
||
# numbers/words listed as bare lines after the instruction line
|
||
return max(il, len(qn), len(qp), 1)
|
||
if tt == "match_letters":
|
||
# items numbered in the query if present, else numbered in the context
|
||
return len(qn) or len(cn) or il or 1
|
||
if tt == "fill_blanks":
|
||
return max(len(qp), len(qn), len(cp), il, 1)
|
||
# translation / default
|
||
n = max(len(qn), len(qp))
|
||
if n <= 1:
|
||
# bare item lines in the query, else blanks enumerated in the context
|
||
n = il if il >= 1 else max(len(cp), 1)
|
||
return max(n, 1)
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Prompt building (task-type aware)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
SYSTEM_BASE = (
|
||
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
||
"problems. Every problem is fully self-contained: use ONLY the data, "
|
||
"examples and hints given to deduce the grammar, vocabulary and rules of "
|
||
"the language. Never rely on outside knowledge of the language — work the "
|
||
"pattern out from the given data. First reason it out silently, then answer "
|
||
"every numbered item.\n"
|
||
"OUTPUT FORMAT: respond with ONLY a JSON array of strings — one string per "
|
||
"numbered item, in the SAME order as the query. No keys, no numbering, no "
|
||
"commentary, nothing outside the array."
|
||
)
|
||
|
||
_TASK_GUIDANCE = {
|
||
"translation": (
|
||
"Each answer is the translation of that item, using the vocabulary and "
|
||
"grammar you deduced from the data. Answer in the language the query "
|
||
"names: 'into English' -> English; 'into <language>' -> that language. "
|
||
"Output only the translated text."
|
||
),
|
||
"fill_blanks": (
|
||
"Work out the rule from the complete rows, then fill each blank with the "
|
||
"missing form IN THE LANGUAGE BEING ANALYSED (the non-English / target "
|
||
"language) — never its English meaning. Output only that form."
|
||
),
|
||
"match_letters": (
|
||
"For each numbered item, output ONLY the single UPPERCASE LETTER "
|
||
"(A, B, C, ...) of its correct match. Never output the matched word, its "
|
||
"translation, or any text other than the letter."
|
||
),
|
||
"text_to_num": (
|
||
"Work out the number system from the examples, then COMPUTE each value "
|
||
'and write it in digits (e.g. "285"). Do NOT copy a number from the '
|
||
"examples — derive the value for each new item. Output only the digits."
|
||
),
|
||
"num_to_text": (
|
||
"Work out the number system from the examples, then CONSTRUCT each number "
|
||
"as words in the target (task) language using those rules. Do NOT copy an "
|
||
"example — build the form for the given number. Output only those words."
|
||
),
|
||
}
|
||
_TASK_GUIDANCE_DEFAULT = (
|
||
"Give exactly what each numbered item asks for, in the form the query "
|
||
"requests."
|
||
)
|
||
|
||
|
||
def task_guidance(task_type):
|
||
return _TASK_GUIDANCE.get((task_type or "").strip(), _TASK_GUIDANCE_DEFAULT)
|
||
|
||
|
||
SYSTEM_COT = (
|
||
"You are an expert solver of International Linguistics Olympiad (IOL) "
|
||
"problems. Every problem is self-contained: use ONLY the given data to deduce "
|
||
"the language's rules — never outside knowledge of the language. Reason "
|
||
"step by step to work out the pattern, then give your answers.\n"
|
||
"FINAL LINE: after your reasoning, output ONLY a JSON array of strings — one "
|
||
"per numbered item, in order — as the very last thing in your reply."
|
||
)
|
||
|
||
|
||
# One compact, fully synthetic worked example per task type. These teach the
|
||
# deduce-then-answer pattern and the terse JSON output form WITHOUT using any
|
||
# real IOL/Linguini data (no contamination). Each answer is verified correct.
|
||
FEWSHOT = {
|
||
"translation": [{
|
||
"user": ("Data from the language Nuu:\nka mi = I see\nka tu = you see\n"
|
||
"lo mi = I go\nka mi ne = I saw\n\nTranslate into English:\n"
|
||
"1. lo tu\n2. ka tu ne"),
|
||
"assistant": '["you go", "you saw"]'}],
|
||
"text_to_num": [{
|
||
"user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before "
|
||
"ones: 'ka ta' = 11.\n\nWrite in digits:\n1. ka ba\n2. ta"),
|
||
"assistant": '["12", "1"]'}],
|
||
"num_to_text": [{
|
||
"user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before "
|
||
"ones: 'ka ta' = 11.\n\nWrite in Zaz:\n1. 12\n2. 10"),
|
||
"assistant": '["ka ba", "ka"]'}],
|
||
"fill_blanks": [{
|
||
"user": ("Verb forms:\nsing | singem | to sing\ndance | (1) | to dance\n\n"
|
||
"Fill the blanks (1)."),
|
||
"assistant": '["dancem"]'}],
|
||
"match_letters": [{
|
||
"user": ("Clues: 'bo' occurs with fire, 'ka' with stone, 'mi' with water.\n"
|
||
"Words:\n1. mi\n2. bo\n3. ka\nMeanings:\nA. stone\nB. water\nC. fire\n\n"
|
||
"Determine the correct correspondences."),
|
||
"assistant": '["B", "C", "A"]'}],
|
||
}
|
||
|
||
|
||
def build_messages(context, query, task_type, n, cot=False, fewshot=False):
|
||
"""Return chat messages. ``n`` = required answer count. ``cot`` = let the
|
||
model reason before the final JSON array. ``fewshot`` = prepend a synthetic
|
||
worked example for this task type (demonstrates the deduce-then-answer
|
||
pattern and terse output form)."""
|
||
base = SYSTEM_COT if cot else SYSTEM_BASE
|
||
system = (
|
||
f"{base}\n{task_guidance(task_type)}\n"
|
||
f"The JSON array must have EXACTLY {n} string{'s' if n != 1 else ''}."
|
||
)
|
||
tail = (
|
||
f"Answer all {n} item{'s' if n != 1 else ''} as a JSON array of "
|
||
f"{n} string{'s' if n != 1 else ''}, in order"
|
||
+ (", after your step-by-step reasoning." if cot else ".")
|
||
)
|
||
user = f"{(context or '').strip()}\n\n{(query or '').strip()}\n\n{tail}"
|
||
|
||
msgs = [{"role": "system", "content": system}]
|
||
if fewshot:
|
||
for ex in FEWSHOT.get((task_type or "").strip(), []):
|
||
msgs.append({"role": "user", "content": ex["user"]})
|
||
msgs.append({"role": "assistant", "content": ex["assistant"]})
|
||
msgs.append({"role": "user", "content": user})
|
||
return msgs
|
||
|
||
|
||
def max_new_tokens_for(task_type, n, cot=False, cap=1024):
|
||
"""Heuristic generation budget. ``cot`` adds room for reasoning (bounded by
|
||
``cap`` to protect the 30-min limit on a T4)."""
|
||
per_item = 96 if (task_type or "") == "translation" else 40
|
||
answer_room = 160 + per_item * max(1, n)
|
||
if cot:
|
||
return int(min(cap, answer_room + 640))
|
||
return int(min(cap, answer_room))
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Answer parsing (robust, multi-layer)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
_FENCE = re.compile(r"```(?:json|python)?\s*(.*?)```", re.DOTALL | re.IGNORECASE)
|
||
_LINE_NUM = re.compile(r"^\s*[\(\[]?(\d{1,3})[\).\]:]\s*(.*\S)?\s*$")
|
||
_PREFIX = re.compile(r"^\s*(?:answers?|output|result)\s*[:\-]\s*", re.IGNORECASE)
|
||
|
||
|
||
def _to_str(x):
|
||
if x is None:
|
||
return ""
|
||
if isinstance(x, (list, tuple)):
|
||
# A nested item (e.g. multiple accepted forms) — join readably.
|
||
return " ".join(_to_str(e) for e in x)
|
||
return str(x)
|
||
|
||
|
||
def _strip_quotes(s):
|
||
s = s.strip()
|
||
if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`":
|
||
s = s[1:-1].strip()
|
||
return s
|
||
|
||
|
||
def _clean(s):
|
||
return _strip_quotes(_to_str(s).strip())
|
||
|
||
|
||
def _strip_fences(text):
|
||
m = _FENCE.search(text or "")
|
||
return m.group(1) if m else (text or "")
|
||
|
||
|
||
def _balanced_arrays(text):
|
||
"""Yield every top-level [...] substring (handles nesting)."""
|
||
depth, start = 0, -1
|
||
for i, ch in enumerate(text):
|
||
if ch == "[":
|
||
if depth == 0:
|
||
start = i
|
||
depth += 1
|
||
elif ch == "]" and depth > 0:
|
||
depth -= 1
|
||
if depth == 0 and start >= 0:
|
||
yield text[start : i + 1]
|
||
|
||
|
||
def extract_json_array(text):
|
||
"""Return the LAST bracketed list that parses, as a list of strings, else None.
|
||
|
||
'Last' matters for reasoning models: they emit <think>...</think> (often
|
||
containing brackets) and then the final answer array — we want that final one.
|
||
"""
|
||
if not text:
|
||
return None
|
||
for frag in reversed(list(_balanced_arrays(text))):
|
||
for parser in (json.loads, ast.literal_eval):
|
||
try:
|
||
value = parser(frag)
|
||
except Exception:
|
||
continue
|
||
if isinstance(value, list):
|
||
return [_clean(x) for x in value]
|
||
return None
|
||
|
||
|
||
def parse_numbered_lines(text):
|
||
"""Parse 'N. answer' / 'N) answer' / '(N) answer' lines -> ordered answers."""
|
||
found = {}
|
||
for line in (text or "").splitlines():
|
||
m = _LINE_NUM.match(line)
|
||
if m and m.group(2):
|
||
found[int(m.group(1))] = _clean(m.group(2))
|
||
if not found:
|
||
return []
|
||
return [found[k] for k in sorted(found)]
|
||
|
||
|
||
def _fit(values, n):
|
||
"""Force ``values`` to exactly ``n`` entries (truncate / pad with '')."""
|
||
values = list(values)[:n]
|
||
values += [""] * (n - len(values))
|
||
return values
|
||
|
||
|
||
def parse_answers(text, n):
|
||
"""Turn raw model output into exactly ``n`` cleaned answer strings.
|
||
|
||
Layers: JSON array -> numbered lines -> plain non-empty lines. Always
|
||
returns a list of length ``n``; extras are dropped, shortfalls padded so no
|
||
item is silently missing (the scorer aligns predictions by position).
|
||
"""
|
||
n = max(1, int(n))
|
||
inner = _strip_fences(text)
|
||
# Reasoning models wrap their scratch-work in <think>...</think>; the answer
|
||
# follows the closing tag. Keep only what comes after it.
|
||
if "</think>" in inner:
|
||
inner = inner.rsplit("</think>", 1)[1]
|
||
|
||
arr = extract_json_array(inner)
|
||
if arr:
|
||
return _fit(arr, n)
|
||
|
||
numbered = parse_numbered_lines(inner)
|
||
if numbered:
|
||
return _fit(numbered, n)
|
||
|
||
lines = [_clean(_PREFIX.sub("", ln)) for ln in inner.splitlines()]
|
||
lines = [ln for ln in lines if ln]
|
||
if lines:
|
||
return _fit(lines, n)
|
||
|
||
single = _clean(inner)
|
||
return _fit([single] if single else [], n)
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Self-consistency: majority vote across passes
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def majority_vote(passes, n):
|
||
"""Per-item majority vote across ``passes`` (each a length-n answer list).
|
||
|
||
Votes are counted case/space-insensitively but the winner keeps its original
|
||
casing. Ties break toward the EARLIEST pass — so pass 0 (greedy) acts as a
|
||
floor: confident items keep the greedy answer, uncertain ones adopt the
|
||
consensus. Non-empty answers are preferred over blanks. Always returns n items.
|
||
"""
|
||
n = max(1, int(n))
|
||
out = []
|
||
for i in range(n):
|
||
col = [p[i] for p in passes if i < len(p)]
|
||
counts, first = {}, {}
|
||
for k, ans in enumerate(col):
|
||
key = _to_str(ans).strip().lower()
|
||
counts[key] = counts.get(key, 0) + 1
|
||
if key not in first:
|
||
first[key] = (k, ans)
|
||
# drop the empty-string option unless it is all we have
|
||
nonblank = {k_: v for k_, v in counts.items() if k_ != ""}
|
||
pool = nonblank or counts
|
||
if not pool:
|
||
out.append("")
|
||
continue
|
||
best = max(pool, key=lambda kk: (pool[kk], -first[kk][0]))
|
||
out.append(first[best][1])
|
||
return out
|