"""Pure-Python harness logic for the IOL-AI 2026 submission. Kept free of torch/transformers so it can be unit-tested locally without a GPU or the model. ``script.py`` (which runs on the T4) imports from here; so does ``test_parsing.py``. Both this file and ``script.py`` are uploaded to the submission repo, so ``import iol_harness`` resolves at run time (the repo root is the working directory and on sys.path[0]). """ import ast import json import re # --------------------------------------------------------------------------- # # Item counting # --------------------------------------------------------------------------- # # "17. ", "18) ", ... anywhere after start/space/comma/semicolon, followed by # whitespace or end-of-line. Requires whitespace after the . or ) so decimals # like "3.14" are NOT matched. _NUMBERED = re.compile(r"(?:^|[\s,;])(\d{1,3})[.)](?=\s|$)", re.MULTILINE) # "(1)", "(2)" — parenthesised single number. "(1-2)" is NOT matched. _PAREN = re.compile(r"\((\d{1,3})\)") # Explicit item range in the query, e.g. "(1-9)", "(13–16)", "(1—10)". _RANGE = re.compile(r"\((\d{1,3})\s*[-–—]\s*(\d{1,3})\)") def _distinct(rx, text): return sorted(set(int(x) for x in rx.findall(text or ""))) def _range_count(query): """Count from an explicit "(a-b)" range in the query, else 0.""" best = 0 for a, b in _RANGE.findall(query or ""): a, b = int(a), int(b) if b >= a: best = max(best, b - a + 1) return best def _item_lines(query): """Non-empty lines after the first non-empty (instruction) line.""" lines = [ln for ln in (query or "").splitlines() if ln.strip()] return max(0, len(lines) - 1) def count_items(context, query, task_type=""): """How many answers the problem expects (validated on real Linguini data). Items live in different places by task type: numbered/bare lines in the query, blanks "(N)" or a numbered list in the context, or an explicit "(a-b)" range in the query. Returns an int >= 1. Biased so that when in doubt it does not under-count (the scorer ignores extra predictions but zero-scores any item a short prediction fails to cover). """ tt = (task_type or "").strip() # 1. An explicit range in the query is the strongest, cleanest signal. r = _range_count(query) if r: return r qn, qp = _distinct(_NUMBERED, query), _distinct(_PAREN, query) cn, cp = _distinct(_NUMBERED, context), _distinct(_PAREN, context) il = _item_lines(query) if tt in ("text_to_num", "num_to_text"): # numbers/words listed as bare lines after the instruction line return max(il, len(qn), len(qp), 1) if tt == "match_letters": # items numbered in the query if present, else numbered in the context return len(qn) or len(cn) or il or 1 if tt == "fill_blanks": return max(len(qp), len(qn), len(cp), il, 1) # translation / default n = max(len(qn), len(qp)) if n <= 1: # bare item lines in the query, else blanks enumerated in the context n = il if il >= 1 else max(len(cp), 1) return max(n, 1) # --------------------------------------------------------------------------- # # Prompt building (task-type aware) # --------------------------------------------------------------------------- # SYSTEM_BASE = ( "You are an expert solver of International Linguistics Olympiad (IOL) " "problems. Every problem is fully self-contained: use ONLY the data, " "examples and hints given to deduce the grammar, vocabulary and rules of " "the language. Never rely on outside knowledge of the language — work the " "pattern out from the given data. First reason it out silently, then answer " "every numbered item.\n" "OUTPUT FORMAT: respond with ONLY a JSON array of strings — one string per " "numbered item, in the SAME order as the query. No keys, no numbering, no " "commentary, nothing outside the array." ) _TASK_GUIDANCE = { "translation": ( "Each answer is the translation of that item, using the vocabulary and " "grammar you deduced from the data. Answer in the language the query " "names: 'into English' -> English; 'into ' -> that language. " "Output only the translated text." ), "fill_blanks": ( "Work out the rule from the complete rows, then fill each blank with the " "missing form IN THE LANGUAGE BEING ANALYSED (the non-English / target " "language) — never its English meaning. Output only that form." ), "match_letters": ( "For each numbered item, output ONLY the single UPPERCASE LETTER " "(A, B, C, ...) of its correct match. Never output the matched word, its " "translation, or any text other than the letter." ), "text_to_num": ( "Work out the number system from the examples, then COMPUTE each value " 'and write it in digits (e.g. "285"). Do NOT copy a number from the ' "examples — derive the value for each new item. Output only the digits." ), "num_to_text": ( "Work out the number system from the examples, then CONSTRUCT each number " "as words in the target (task) language using those rules. Do NOT copy an " "example — build the form for the given number. Output only those words." ), } _TASK_GUIDANCE_DEFAULT = ( "Give exactly what each numbered item asks for, in the form the query " "requests." ) def task_guidance(task_type): return _TASK_GUIDANCE.get((task_type or "").strip(), _TASK_GUIDANCE_DEFAULT) SYSTEM_COT = ( "You are an expert solver of International Linguistics Olympiad (IOL) " "problems. Every problem is self-contained: use ONLY the given data to deduce " "the language's rules — never outside knowledge of the language. Reason " "step by step to work out the pattern, then give your answers.\n" "FINAL LINE: after your reasoning, output ONLY a JSON array of strings — one " "per numbered item, in order — as the very last thing in your reply." ) # One compact, fully synthetic worked example per task type. These teach the # deduce-then-answer pattern and the terse JSON output form WITHOUT using any # real IOL/Linguini data (no contamination). Each answer is verified correct. FEWSHOT = { "translation": [{ "user": ("Data from the language Nuu:\nka mi = I see\nka tu = you see\n" "lo mi = I go\nka mi ne = I saw\n\nTranslate into English:\n" "1. lo tu\n2. ka tu ne"), "assistant": '["you go", "you saw"]'}], "text_to_num": [{ "user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before " "ones: 'ka ta' = 11.\n\nWrite in digits:\n1. ka ba\n2. ta"), "assistant": '["12", "1"]'}], "num_to_text": [{ "user": ("Numbers in Zaz: ta = 1, ba = 2, ka = 10. Tens come before " "ones: 'ka ta' = 11.\n\nWrite in Zaz:\n1. 12\n2. 10"), "assistant": '["ka ba", "ka"]'}], "fill_blanks": [{ "user": ("Verb forms:\nsing | singem | to sing\ndance | (1) | to dance\n\n" "Fill the blanks (1)."), "assistant": '["dancem"]'}], "match_letters": [{ "user": ("Clues: 'bo' occurs with fire, 'ka' with stone, 'mi' with water.\n" "Words:\n1. mi\n2. bo\n3. ka\nMeanings:\nA. stone\nB. water\nC. fire\n\n" "Determine the correct correspondences."), "assistant": '["B", "C", "A"]'}], } def build_messages(context, query, task_type, n, cot=False, fewshot=False): """Return chat messages. ``n`` = required answer count. ``cot`` = let the model reason before the final JSON array. ``fewshot`` = prepend a synthetic worked example for this task type (demonstrates the deduce-then-answer pattern and terse output form).""" base = SYSTEM_COT if cot else SYSTEM_BASE system = ( f"{base}\n{task_guidance(task_type)}\n" f"The JSON array must have EXACTLY {n} string{'s' if n != 1 else ''}." ) tail = ( f"Answer all {n} item{'s' if n != 1 else ''} as a JSON array of " f"{n} string{'s' if n != 1 else ''}, in order" + (", after your step-by-step reasoning." if cot else ".") ) user = f"{(context or '').strip()}\n\n{(query or '').strip()}\n\n{tail}" msgs = [{"role": "system", "content": system}] if fewshot: for ex in FEWSHOT.get((task_type or "").strip(), []): msgs.append({"role": "user", "content": ex["user"]}) msgs.append({"role": "assistant", "content": ex["assistant"]}) msgs.append({"role": "user", "content": user}) return msgs def max_new_tokens_for(task_type, n, cot=False, cap=1024): """Heuristic generation budget. ``cot`` adds room for reasoning (bounded by ``cap`` to protect the 30-min limit on a T4).""" per_item = 96 if (task_type or "") == "translation" else 40 answer_room = 160 + per_item * max(1, n) if cot: return int(min(cap, answer_room + 640)) return int(min(cap, answer_room)) # --------------------------------------------------------------------------- # # Answer parsing (robust, multi-layer) # --------------------------------------------------------------------------- # _FENCE = re.compile(r"```(?:json|python)?\s*(.*?)```", re.DOTALL | re.IGNORECASE) _LINE_NUM = re.compile(r"^\s*[\(\[]?(\d{1,3})[\).\]:]\s*(.*\S)?\s*$") _PREFIX = re.compile(r"^\s*(?:answers?|output|result)\s*[:\-]\s*", re.IGNORECASE) def _to_str(x): if x is None: return "" if isinstance(x, (list, tuple)): # A nested item (e.g. multiple accepted forms) — join readably. return " ".join(_to_str(e) for e in x) return str(x) def _strip_quotes(s): s = s.strip() if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'`": s = s[1:-1].strip() return s def _clean(s): return _strip_quotes(_to_str(s).strip()) def _strip_fences(text): m = _FENCE.search(text or "") return m.group(1) if m else (text or "") def _balanced_arrays(text): """Yield every top-level [...] substring (handles nesting).""" depth, start = 0, -1 for i, ch in enumerate(text): if ch == "[": if depth == 0: start = i depth += 1 elif ch == "]" and depth > 0: depth -= 1 if depth == 0 and start >= 0: yield text[start : i + 1] def extract_json_array(text): """Return the LAST bracketed list that parses, as a list of strings, else None. 'Last' matters for reasoning models: they emit ... (often containing brackets) and then the final answer array — we want that final one. """ if not text: return None for frag in reversed(list(_balanced_arrays(text))): for parser in (json.loads, ast.literal_eval): try: value = parser(frag) except Exception: continue if isinstance(value, list): return [_clean(x) for x in value] return None def parse_numbered_lines(text): """Parse 'N. answer' / 'N) answer' / '(N) answer' lines -> ordered answers.""" found = {} for line in (text or "").splitlines(): m = _LINE_NUM.match(line) if m and m.group(2): found[int(m.group(1))] = _clean(m.group(2)) if not found: return [] return [found[k] for k in sorted(found)] def _fit(values, n): """Force ``values`` to exactly ``n`` entries (truncate / pad with '').""" values = list(values)[:n] values += [""] * (n - len(values)) return values def parse_answers(text, n): """Turn raw model output into exactly ``n`` cleaned answer strings. Layers: JSON array -> numbered lines -> plain non-empty lines. Always returns a list of length ``n``; extras are dropped, shortfalls padded so no item is silently missing (the scorer aligns predictions by position). """ n = max(1, int(n)) inner = _strip_fences(text) # Reasoning models wrap their scratch-work in ...; the answer # follows the closing tag. Keep only what comes after it. if "" in inner: inner = inner.rsplit("", 1)[1] arr = extract_json_array(inner) if arr: return _fit(arr, n) numbered = parse_numbered_lines(inner) if numbered: return _fit(numbered, n) lines = [_clean(_PREFIX.sub("", ln)) for ln in inner.splitlines()] lines = [ln for ln in lines if ln] if lines: return _fit(lines, n) single = _clean(inner) return _fit([single] if single else [], n) # --------------------------------------------------------------------------- # # Self-consistency: majority vote across passes # --------------------------------------------------------------------------- # def majority_vote(passes, n): """Per-item majority vote across ``passes`` (each a length-n answer list). Votes are counted case/space-insensitively but the winner keeps its original casing. Ties break toward the EARLIEST pass — so pass 0 (greedy) acts as a floor: confident items keep the greedy answer, uncertain ones adopt the consensus. Non-empty answers are preferred over blanks. Always returns n items. """ n = max(1, int(n)) out = [] for i in range(n): col = [p[i] for p in passes if i < len(p)] counts, first = {}, {} for k, ans in enumerate(col): key = _to_str(ans).strip().lower() counts[key] = counts.get(key, 0) + 1 if key not in first: first[key] = (k, ans) # drop the empty-string option unless it is all we have nonblank = {k_: v for k_, v in counts.items() if k_ != ""} pool = nonblank or counts if not pool: out.append("") continue best = max(pool, key=lambda kk: (pool[kk], -first[kk][0])) out.append(first[best][1]) return out