""" Shared prompt and answer-parsing logic for the IOL submission. Strategy: direct task-specific prompting with robust answer parsing. The model outputs answers directly (one per line). Qwen3 models may use ... tags for reasoning, which we extract for the explanation column and then parse the final answer. CRITICAL FIX from v1: The _looks_like_analysis filter is REMOVED. It was dropping ~5% of correct translation answers that started with common English words (The, We, There, Where, etc.). """ import re USER_TEMPLATE = """\ {context} {query}""" # --- Task-specific system prompts (direct, no CoT) --- _PROMPTS = { "translation": """\ You solve International Linguistics Olympiad translation problems. \ Bilingual examples from an unfamiliar language are given. \ Deduce the vocabulary and grammar, then translate each numbered item. Output format (REQUIRED): Step 1: Identify the source word for each numbered item and look up its meaning from the examples. Step 2: Note any grammar (case, tense, agreement) that must be applied. Step 3: Compose the EXACT target-language form, preserving notation like You_{sg}, You_{pl}. [translation 1] [translation 2] ... Rules: - Work out word meanings, affixes, and word order from the examples. - The section is the ONLY thing that is scored — be exact. - After , output ONLY the translations — nothing else. - One translation per line, in order, no numbering, no labels, no commentary. - Preserve notation like You_{sg}, You_{pl} exactly as used in the examples. - Always give your best guess — never leave an item blank.""", "fill_blanks": """\ You solve International Linguistics Olympiad fill-in-the-blank problems. \ Morphological or syntactic paradigms from an unfamiliar language are given. \ Deduce the pattern, then fill each blank. Output format (REQUIRED): Step 1: Identify the pattern (prefix, suffix, stem change, or syntactic rule). Step 2: Apply the pattern to each blank, noting any irregularities. Step 3: Write the EXACT filled form, preserving notation like 1sg, 2pl. [form 1] [form 2] ... Rules: - Identify prefixes, suffixes, stem changes, or syntactic patterns from the examples. - The section is the ONLY thing that is scored — be exact. - After , output ONLY the filled forms — nothing else. - One form per line, in order, no numbering, no labels, no commentary. - Always give your best guess — never leave an item blank.""", "text_to_num": """\ You are an expert linguistic analyst solving International Linguistics Olympiad number transliteration problems. \ You are given number words in an unfamiliar language with their values. \ Work out the number system (bases, multipliers, additives), then convert. Rules: - Identify the base system (e.g. base-5, base-10, base-20) and how numbers combine. - Do NOT use outside knowledge. Everything you need is in the problem data. - Answer every numbered item, one per line, in order. - Give ONLY the numeral in digits (e.g. 111), nothing else. No words, no explanation. - If unsure, give your best guess rather than leaving a blank.""", "num_to_text": """\ You are an expert linguistic analyst solving International Linguistics Olympiad number transliteration problems. \ You are given number words in an unfamiliar language with their values. \ Work out the number system (bases, multipliers, additives), then convert. Rules: - Identify the base system (e.g. base-5, base-10, base-20) and how numbers combine. - Do NOT use outside knowledge. Everything you need is in the problem data. - Answer every numbered item, one per line, in order. - Give ONLY the number in the target language's words, nothing else. No numerals, no explanation. - If unsure, give your best guess rather than leaving a blank.""", "match_letters": """\ You match words to their translations in an unfamiliar language. \ The words and translations are given in arbitrary order. You MUST output ONLY single capital letters, one per line, in the order of the numbered items. \ Do NOT explain your reasoning. Do NOT output words or sentences. \ Output ONLY the letters (A, B, C, ...), one per line. Example output: A C B D Now match each item. Output ONLY letters, nothing else.""", # Generic task type used as fallback for unknown types "_generic": """\ You solve International Linguistics Olympiad problems. \ Answer every numbered item in the query, one per line, in order, with no numbering, \ no bullet points, no commentary, no explanations — only the answer itself. \ If unsure, give your best guess rather than leaving a blank. \ Always emit exactly one answer per numbered item, in the same order as the query.""", } # Fallback for unknown task types _DEFAULT_PROMPT = """\ You are an expert linguistic analyst competing in the International Linguistics Olympiad. \ You are given data from a language you have never seen, plus enough examples to deduce its rules. \ Answer every numbered item in the query. Put each answer on its own line, \ in order, with no numbering, no bullet points, and no extra text. \ Always emit exactly one answer per numbered item, in the same order as the query. \ Partial credit is given for roughly-right answers, so always give your best guess — \ never leave an item blank.""" # Legacy aliases SYSTEM_PROMPT = _DEFAULT_PROMPT SYSTEM_PROMPT_SIMPLE = _DEFAULT_PROMPT def get_system_prompt(task_type: str = "") -> str: """Select the best system prompt for a given task type.""" if task_type and task_type in _PROMPTS: return _PROMPTS[task_type] return _DEFAULT_PROMPT def count_query_items(query: str) -> int: """Estimate how many numbered items are in the query.""" n = len(re.findall(r"^\s*\d+[\.\)]", query, re.MULTILINE)) return n def parse_answers(text: str, n_expected: int = 0, task_type: str = "") -> list[str]: """Extract answers from model output, handling various formats. Strategies (tried in order): 1. Content after tag (Qwen3 thinking mode) 2. Content between ... tags 3. Content after tag 4. Numbered lines (1. foo, 2. bar) stripped of numbering 5. All non-empty lines (last resort) IMPORTANT: We do NOT filter "analysis-like" lines. The v1 parser had a _looks_like_analysis filter that dropped ~5% of correct translation answers (e.g. "We shuddered.", "The woman is afraid of my dog.") because they started with common English words. That filter is removed. """ answers: list[str] = [] # 0. If Qwen3 thinking mode, extract content after think_split = re.split(r"", text, flags=re.DOTALL) if len(think_split) > 1: text = think_split[-1].strip() # 1. Try ... tags ans_match = re.search(r"\s*(.*?)\s*", text, re.DOTALL) if ans_match: raw = ans_match.group(1) answers = [ln.strip() for ln in raw.splitlines() if ln.strip()] # 2. Try "Final answer:" / "Answer:" section (concise CoT formats) if not answers: # Match "Final answer:" or "Answer:" possibly followed by content; # capture everything after, stopping at any next section header. ans_match = re.search( r"(?:^|\n)\s*(?:Final\s+)?[Aa]nswers?\s*:\s*\n(.*?)(?:\n\s*(?:Reasoning|Explanation|Analysis|Final)\s*:|$)", text, re.DOTALL, ) if ans_match: tail = ans_match.group(1).strip() if tail: answers = [ln.strip() for ln in tail.splitlines() if ln.strip()] # 3. Try content after tag if not answers: post_analysis = re.split(r"", text, flags=re.DOTALL) if len(post_analysis) > 1: tail = post_analysis[-1].strip() if tail: answers = [ln.strip() for ln in tail.splitlines() if ln.strip()] # 4. Try numbered lines, stripping numbering if not answers: lines = [ln.strip() for ln in text.splitlines() if ln.strip()] numbered = [] for ln in lines: m = re.match(r"^\d+[\.\)]\s*(.+)", ln) if m: numbered.append(m.group(1).strip()) if numbered: answers = numbered # 5. Last resort: all non-empty lines, skip obvious markdown/headers # NOT skipping analysis prose — that was the v1 bug. if not answers: lines = [ln.strip() for ln in text.splitlines() if ln.strip()] filtered = [] for ln in lines: if ln.startswith(("#", "- ", "* ", "**", "|", "<", "```")): continue filtered.append(ln) if filtered: answers = filtered # 5. Clean up: strip numbering, bullets, markdown cleaned = [] for a in answers: a = re.sub(r"^\d+[\.\)]\s*", "", a) # strip "1. " prefix a = a.strip("- *•\t ") a = re.sub(r"\*{1,2}(.+?)\*{1,2}", r"\1", a) # strip markdown bold/italic if a: cleaned.append(a) # 6. Task-specific post-processing if task_type == "match_letters": cleaned = _extract_letters(cleaned, n_expected) elif task_type == "text_to_num": cleaned = _extract_numbers(cleaned, n_expected) # 7. Ensure at least one answer if not cleaned: cleaned = [""] # 8. Pad if we got fewer than expected if n_expected and len(cleaned) < n_expected: cleaned.extend([""] * (n_expected - len(cleaned))) # 9. Trim if we got more than expected if n_expected and len(cleaned) > n_expected: cleaned = cleaned[:n_expected] return cleaned def extract_analysis(text: str) -> str: """Extract reasoning for the explanation column (human jury track). For Qwen3 thinking mode, extracts the ... content. For the new concise CoT format, extracts the "Reasoning:" line. For direct-prompt mode, returns a brief note. """ # Try tags (Qwen3) m = re.search(r"(.*?)", text, re.DOTALL) if m: analysis = m.group(1).strip() if len(analysis) > 500: analysis = analysis[:497] + "..." return analysis # Try "Reasoning:" line (new concise CoT format) m = re.search( r"(?:^|\n)\s*Reasoning\s*:\s*(.+?)(?:\n\s*(?:Final\s+)?[Aa]nswers?\s*:|$)", text, re.DOTALL | re.IGNORECASE, ) if m: analysis = m.group(1).strip() if len(analysis) > 500: analysis = analysis[:497] + "..." return analysis # Try tags m = re.search(r"\s*(.*?)\s*", text, re.DOTALL) if m: analysis = m.group(1).strip() if len(analysis) > 500: analysis = analysis[:497] + "..." return analysis # For direct prompts, provide a minimal explanation # If there's thinking content (text before ), use it think_split = re.split(r"", text, maxsplit=1) if len(think_split) > 1 and think_split[0].strip(): analysis = think_split[0].strip() if len(analysis) > 500: analysis = analysis[:497] + "..." return analysis return "Direct translation using deduced linguistic patterns from examples." def _extract_letters(answers: list[str], n_expected: int) -> list[str]: """For match_letters: extract single capital letters from the output.""" letters = [] for a in answers: m = re.search(r"\b([A-Z])\b", a) if m: letters.append(m.group(1)) elif len(a) == 1 and a.isalpha(): letters.append(a.upper()) elif len(a) == 2 and a[1].isalpha() and a[0].isdigit(): letters.append(a[1].upper()) else: m = re.search(r"([A-Z])(?:\s*[->:\|]\s*|\s*$)", a) if m: letters.append(m.group(1)) else: letters.append(a) if n_expected and len(letters) < n_expected: letters.extend(["A"] * (n_expected - len(letters))) return letters def _extract_numbers(answers: list[str], n_expected: int) -> list[str]: """For text_to_num: extract the numeral from the answer.""" numbers = [] for a in answers: m = re.search(r"(\d+(?:\s*[-+^=]\s*\d+)*)", a) if m: numbers.append(m.group(1).replace(" ", "")) else: numbers.append(a) return numbers