385 lines
15 KiB
Python
385 lines
15 KiB
Python
"""Parse Linguini puzzles: normalize text, parse the context (pipe tables,
|
||
numbered/lettered lists, pairs) and the query into answerable items."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import unicodedata
|
||
from dataclasses import dataclass, field
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
_ITEM_PREFIX = re.compile(r"^\s*\(?(\d{1,3})[\.\)]\s+")
|
||
_LETTER_PREFIX = re.compile(r"^\s*\(?([A-Z])[\.\)]\s+")
|
||
_BLANK_MARK = re.compile(r"\((\d{1,3})\)")
|
||
_BLANK_LINE = re.compile(r"_{2,}|…|\.{4,}")
|
||
|
||
# non-pipe two-side separators, tried in order on non-table lines
|
||
_SEPARATORS = [
|
||
("tab", re.compile(r"\t+")),
|
||
("equals", re.compile(r"\s+=\s+")),
|
||
("emdash", re.compile(r"\s+—\s+")),
|
||
("endash", re.compile(r"\s+–\s+")),
|
||
("arrow", re.compile(r"\s*(?:->|→)\s*")),
|
||
("hyphen", re.compile(r"\s+-\s+")),
|
||
("means", re.compile(r"\s+means\s+", re.IGNORECASE)),
|
||
]
|
||
|
||
# common work-language names (queries say "Translate into English:")
|
||
_WORK_LANG_NAMES = {
|
||
"eng": "english", "fra": "french", "spa": "spanish", "por": "portuguese",
|
||
"rus": "russian", "deu": "german",
|
||
}
|
||
|
||
|
||
def normalize(text: str) -> str:
|
||
"""NFC-normalize, unify exotic whitespace/quotes. Keeps diacritics, tone
|
||
marks, case, and punctuation (EM comparison is punctuation-sensitive)."""
|
||
if text is None:
|
||
return ""
|
||
t = unicodedata.normalize("NFC", str(text))
|
||
t = t.replace(" ", " ")
|
||
t = re.sub(r"[ \t]+", " ", t)
|
||
return t.strip()
|
||
|
||
|
||
def tokenize(s: str) -> List[str]:
|
||
"""Unicode word tokenization. Keeps combining marks, word-internal
|
||
apostrophes/hyphens, and subscript/superscript markers (tone letters,
|
||
person markers like you_{sg})."""
|
||
s = normalize(s)
|
||
return re.findall(r"[^\s,;.!?()\[\]\"«»|]+", s)
|
||
|
||
|
||
def strip_punct(tok: str) -> str:
|
||
return tok.strip(",;.!?()[]\"«»").strip()
|
||
|
||
|
||
@dataclass
|
||
class Pair:
|
||
src: str # task-language side by convention
|
||
tgt: str # work-language side (gloss/translation)
|
||
sep: str = ""
|
||
line_no: int = -1
|
||
label: str = "" # numbered prefix if the line carried one
|
||
|
||
|
||
@dataclass
|
||
class QueryItem:
|
||
number: str # label as it appeared ("17", "3", "") — "" for bare lines
|
||
text: str
|
||
direction: Optional[str] = None # "to_task" | "to_work" | None
|
||
has_blank: bool = False
|
||
row: Optional[List[str]] = None # for table-blank items: full row cells
|
||
blank_col: Optional[int] = None # which cell holds this item's (k) marker
|
||
|
||
|
||
@dataclass
|
||
class Puzzle:
|
||
id: str
|
||
context: str
|
||
query: str
|
||
work_lang: str = ""
|
||
task_lang: str = ""
|
||
task_type: str = ""
|
||
eval_type: str = ""
|
||
pairs: List[Pair] = field(default_factory=list)
|
||
items: List[QueryItem] = field(default_factory=list)
|
||
hints: List[str] = field(default_factory=list)
|
||
tables: List[List[List[str]]] = field(default_factory=list) # blocks of rows of cells
|
||
numbered: Dict[str, str] = field(default_factory=dict) # "1" -> form (list contexts)
|
||
lettered: Dict[str, str] = field(default_factory=dict) # "A" -> meaning
|
||
|
||
|
||
def _split_cells(line: str) -> List[str]:
|
||
return [c.strip() for c in line.split("|")]
|
||
|
||
|
||
def _strip_item_prefix(line: str) -> Tuple[str, str]:
|
||
"""Returns (label, rest). Label may be a number or capital letter."""
|
||
m = _ITEM_PREFIX.match(line)
|
||
if m:
|
||
return m.group(1), line[m.end():].strip()
|
||
m = _LETTER_PREFIX.match(line)
|
||
if m:
|
||
return m.group(1), line[m.end():].strip()
|
||
return "", line.strip()
|
||
|
||
|
||
def _looks_header(cells: List[str]) -> bool:
|
||
"""A table header names languages/columns: 'Proto-Chamic | Tsat | meaning'."""
|
||
if len(cells) < 2:
|
||
return False
|
||
tail = cells[-1].lower()
|
||
if tail in ("meaning", "meanings", "translation", "translations", "english",
|
||
"value", "values", "gloss"):
|
||
return True
|
||
# all cells capitalized single-ish words with no digits — likely names
|
||
ok = 0
|
||
for c in cells:
|
||
if c and not any(ch.isdigit() for ch in c) and c[0].isupper() and len(c.split()) <= 3:
|
||
ok += 1
|
||
return ok == len(cells) and len(cells) >= 3
|
||
|
||
|
||
def parse_context(ctx: str) -> Tuple[List[Pair], List[str], List[List[List[str]]], Dict[str, str], Dict[str, str]]:
|
||
"""Parse context into (pairs, hints, tables, numbered, lettered)."""
|
||
pairs: List[Pair] = []
|
||
hints: List[str] = []
|
||
tables: List[List[List[str]]] = []
|
||
numbered: Dict[str, str] = {}
|
||
lettered: Dict[str, str] = {}
|
||
|
||
cur_table: List[List[str]] = []
|
||
for i, raw in enumerate(str(ctx).splitlines()):
|
||
line = normalize(raw)
|
||
if not line:
|
||
if cur_table:
|
||
tables.append(cur_table)
|
||
cur_table = []
|
||
continue
|
||
label, body = _strip_item_prefix(line)
|
||
|
||
if "|" in body:
|
||
cells = _split_cells(body)
|
||
if _looks_header(cells) and not cur_table:
|
||
hints.append(line)
|
||
continue
|
||
cur_table.append(cells)
|
||
has_blank = bool(_BLANK_MARK.search(body))
|
||
if len(cells) >= 2 and cells[0] and cells[-1] and not has_blank:
|
||
pairs.append(Pair(src=cells[0], tgt=cells[-1], sep="pipe",
|
||
line_no=i, label=label))
|
||
if label and not has_blank:
|
||
numbered[label] = cells[0]
|
||
continue
|
||
|
||
if cur_table:
|
||
tables.append(cur_table)
|
||
cur_table = []
|
||
|
||
# non-pipe separators (= , — , tab ...)
|
||
matched = False
|
||
for name, rx in _SEPARATORS:
|
||
parts = rx.split(body, maxsplit=1)
|
||
if len(parts) == 2 and parts[0].strip() and parts[1].strip():
|
||
pairs.append(Pair(src=parts[0].strip(), tgt=parts[1].strip(),
|
||
sep=name, line_no=i, label=label))
|
||
if label:
|
||
# the full line is the referable entry ("equalities (1-9)")
|
||
numbered[label] = body
|
||
matched = True
|
||
break
|
||
if matched:
|
||
continue
|
||
|
||
# single-column list entries (match_letters forms/meanings)
|
||
if label:
|
||
if label.isdigit():
|
||
numbered[label] = body
|
||
else:
|
||
lettered[label] = body
|
||
continue
|
||
|
||
hints.append(line)
|
||
|
||
if cur_table:
|
||
tables.append(cur_table)
|
||
return pairs, hints, tables, numbered, lettered
|
||
|
||
|
||
_INSTRUCTION_VERBS = (
|
||
r"(translate|fill|write|spell|determine|give|complete|convert|match|answer|"
|
||
r"say|pair|transcribe|provide|express|render|decipher|find|identify|"
|
||
r"choose|select|here|below|these|the following)"
|
||
)
|
||
_INSTRUCTION_RX = re.compile(r"^" + _INSTRUCTION_VERBS + r"\b", re.IGNORECASE)
|
||
_INSTRUCTION_ANY_RX = re.compile(r"\b" + _INSTRUCTION_VERBS + r"\b", re.IGNORECASE)
|
||
|
||
|
||
def _is_instruction(line: str) -> bool:
|
||
"""Instruction lines are work-language imperatives ("Translate into X:").
|
||
Matching is verb-anchored — a bare trailing colon is NOT enough, because
|
||
task-language forms can end in ':' (length marks: "si teŋku bugdiŋi:").
|
||
A line that ends with ':' AND contains an instruction verb anywhere is
|
||
also an instruction ("In Drehu tusi is 'book'. Translate from Drehu:")."""
|
||
line = (line or "").strip()
|
||
if not line:
|
||
return False
|
||
if _BLANK_MARK.search(line) or "|" in line:
|
||
return False
|
||
if _INSTRUCTION_RX.match(line):
|
||
return True
|
||
return line.endswith(":") and bool(_INSTRUCTION_ANY_RX.search(line))
|
||
|
||
|
||
def parse_query(query: str) -> Tuple[List[QueryItem], List[str]]:
|
||
"""Split query into answerable items + instruction lines.
|
||
|
||
Item sources, in the order encountered:
|
||
- (k)-markers inside lines (usually pipe rows): one item per marker, with
|
||
the row cells and blank column recorded;
|
||
- numbered lines "17. ..." (numbering may continue the context's);
|
||
- bare non-instruction lines: one item per line.
|
||
"""
|
||
text = str(query or "")
|
||
items: List[QueryItem] = []
|
||
instructions: List[str] = []
|
||
_TERMINAL = (".", "!", "?", ":", ";", '"', "”", "’")
|
||
|
||
for raw in text.splitlines():
|
||
line = normalize(raw)
|
||
if not line:
|
||
continue
|
||
marks = _BLANK_MARK.findall(line)
|
||
if marks:
|
||
cells = _split_cells(line) if "|" in line else [line]
|
||
for k in marks:
|
||
blank_col = next(
|
||
(ci for ci, c in enumerate(cells) if f"({k})" in c), None)
|
||
items.append(QueryItem(
|
||
number=k, text=line, has_blank=True,
|
||
row=cells if len(cells) > 1 else None, blank_col=blank_col))
|
||
continue
|
||
if "|" in line:
|
||
label, body = _strip_item_prefix(line)
|
||
if label:
|
||
# numbered table row = one item; the answer fills whichever
|
||
# column the context table has that this row lacks
|
||
items.append(QueryItem(number=label, text=body,
|
||
row=_split_cells(body)))
|
||
else:
|
||
instructions.append(line) # header/echo row
|
||
continue
|
||
label, body = _strip_item_prefix(line)
|
||
if label:
|
||
items.append(QueryItem(number=label, text=body,
|
||
has_blank=bool(_BLANK_LINE.search(body))))
|
||
continue
|
||
if _is_instruction(line):
|
||
instructions.append(line)
|
||
continue
|
||
if items and items[-1].number and not items[-1].text.rstrip().endswith(_TERMINAL):
|
||
items[-1].text += " " + line # wrapped continuation of a numbered item
|
||
continue
|
||
items.append(QueryItem(number="", text=line,
|
||
has_blank=bool(_BLANK_LINE.search(line))))
|
||
|
||
# when the query has numbered items, stray unnumbered lines around them
|
||
# are notes ("spoken on Bvuŋkaden"), not answerable items
|
||
if any(it.number for it in items):
|
||
items = [it for it in items if it.number]
|
||
# items with numeric labels answer in label order when labels are complete
|
||
if items and all(it.number.isdigit() for it in items):
|
||
items.sort(key=lambda it: int(it.number))
|
||
return items, instructions
|
||
|
||
|
||
def detect_direction(item_text: str, task_material: str, work_material: str,
|
||
instructions: List[str], work_lang: str) -> str:
|
||
"""Per-item direction: does the answer belong to the task language
|
||
('to_task') or the work language ('to_work')?
|
||
|
||
1. Explicit instruction: "into English" (work-lang name) vs "into X".
|
||
2. Script similarity: if the item text overlaps the task-language material
|
||
character-wise, it is task-language text needing analysis (to_work).
|
||
"""
|
||
joined = (" ".join(instructions) + " " + item_text).lower()
|
||
wl_name = _WORK_LANG_NAMES.get(work_lang.split("_")[0][:3].lower(), "")
|
||
m = re.search(r"(?:into|in|to)\s+(?:the\s+)?([A-Za-zÀ-ž’' -]{2,30}?)\s*(?:language)?\s*[:.]", joined + ":")
|
||
if m:
|
||
named = m.group(1).strip().lower()
|
||
if wl_name and wl_name in named:
|
||
return "to_work"
|
||
if named and not any(w in named for w in ("digit", "numeral", "number", "blank")):
|
||
return "to_task"
|
||
sim_task = _char_overlap(item_text, task_material)
|
||
sim_work = _char_overlap(item_text, work_material)
|
||
return "to_work" if sim_task >= sim_work else "to_task"
|
||
|
||
|
||
def _char_overlap(s: str, material: str, n: int = 3) -> float:
|
||
s_ = "".join(s.lower().split())
|
||
m_ = "".join(material.lower().split())
|
||
if len(s_) < n or len(m_) < n:
|
||
return 0.0
|
||
grams = {s_[i : i + n] for i in range(len(s_) - n + 1)}
|
||
hits = sum(1 for g in grams if g in m_)
|
||
return hits / len(grams)
|
||
|
||
|
||
_RANGE_RX = re.compile(r"\((\d{1,3})\s*[–—-]\s*(\d{1,3})\)")
|
||
|
||
|
||
def _items_from_context(p: Puzzle) -> List[QueryItem]:
|
||
"""When the query is instruction-only ("Fill in the blanks (1–14)",
|
||
"Determine the correct correspondences", "Write the equalities (1–9) in
|
||
numerals"), the answerable items live in the CONTEXT: (k) blank markers,
|
||
or the numbered list entries. Last resort: the query itself is one item."""
|
||
rng = _RANGE_RX.search(p.query or "")
|
||
lo, hi = (int(rng.group(1)), int(rng.group(2))) if rng else (None, None)
|
||
|
||
def in_range(k: str) -> bool:
|
||
return lo is None or (k.isdigit() and lo <= int(k) <= hi)
|
||
|
||
ctx_blanks: List[QueryItem] = []
|
||
for raw in str(p.context).splitlines():
|
||
line = normalize(raw)
|
||
for k in _BLANK_MARK.findall(line):
|
||
if not in_range(k):
|
||
continue
|
||
cells = _split_cells(line) if "|" in line else [line]
|
||
blank_col = next((ci for ci, c in enumerate(cells) if f"({k})" in c), None)
|
||
ctx_blanks.append(QueryItem(number=k, text=line, has_blank=True,
|
||
row=cells if len(cells) > 1 else None,
|
||
blank_col=blank_col))
|
||
if ctx_blanks:
|
||
ctx_blanks.sort(key=lambda it: int(it.number))
|
||
return ctx_blanks
|
||
|
||
if p.numbered and (p.task_type == "match_letters" or rng or p.lettered):
|
||
keys = sorted((k for k in p.numbered if in_range(k)), key=int)
|
||
if keys:
|
||
return [QueryItem(number=k, text=p.numbered[k]) for k in keys]
|
||
|
||
q = normalize(p.query)
|
||
return [QueryItem(number="", text=q)] if q else []
|
||
|
||
|
||
def parse_puzzle(row: dict) -> Puzzle:
|
||
"""Build a Puzzle from a CSV/dataset row (id, context, query, work_lang,
|
||
task_lang, task_type, eval_type)."""
|
||
ctx = str(row.get("context", "") or "")
|
||
p = Puzzle(
|
||
id=str(row.get("id", "")),
|
||
context=ctx,
|
||
query=str(row.get("query", "") or ""),
|
||
work_lang=str(row.get("work_lang", "") or ""),
|
||
task_lang=str(row.get("task_lang", "") or ""),
|
||
task_type=str(row.get("task_type", "") or "").strip().lower(),
|
||
eval_type=str(row.get("eval_type", "") or ""),
|
||
)
|
||
p.pairs, p.hints, p.tables, p.numbered, p.lettered = parse_context(ctx)
|
||
p.items, instructions = parse_query(p.query)
|
||
p.hints.extend(instructions)
|
||
|
||
# letter-labelled query entries are answer OPTIONS when digit-labelled
|
||
# items coexist (match tasks list both: "19. form ... S. meaning")
|
||
digit_items = [it for it in p.items if it.number.isdigit()]
|
||
letter_items = [it for it in p.items if it.number and not it.number.isdigit()]
|
||
if digit_items and letter_items:
|
||
for it in letter_items:
|
||
p.lettered[it.number] = it.text
|
||
p.items = digit_items
|
||
|
||
if not p.items:
|
||
p.items = _items_from_context(p)
|
||
|
||
task_material = " ".join(x.src for x in p.pairs) + " " + " ".join(p.numbered.values())
|
||
work_material = " ".join(x.tgt for x in p.pairs) + " " + " ".join(p.lettered.values())
|
||
for it in p.items:
|
||
if it.row is not None:
|
||
continue # table-blank items get direction from their row in the router
|
||
it.direction = detect_direction(it.text, task_material, work_material,
|
||
instructions, p.work_lang)
|
||
return p
|