import os, sys, subprocess, importlib, importlib.metadata, unicodedata from pathlib import Path os.environ.setdefault("HF_HUB_OFFLINE","1"); os.environ.setdefault("TRANSFORMERS_OFFLINE","1") SCRIPT_DIR = Path(__file__).resolve().parent if "__file__" in globals() else Path.cwd() WHEELHOUSE = SCRIPT_DIR / "wheelhouse" if not WHEELHOUSE.is_dir(): WHEELHOUSE = Path("wheelhouse") RUNTIME_DIR = Path("/tmp/qwen3deps") def emergency(reason): try: import pandas as pd, json as j try: ids = pd.read_csv("/tmp/data/test.csv", dtype=str).fillna("")["id"].tolist() except Exception: ids = [] pd.DataFrame([{"id":i,"pred":j.dumps([""]),"explanation":str(reason)[:100]} for i in ids], columns=["id","pred","explanation"]).to_csv("submission.csv", index=False) except Exception: try: open("submission.csv","w").write("id,pred,explanation\n") except Exception: pass try: wheels = [str(WHEELHOUSE / w) for w in os.listdir(WHEELHOUSE) if w.endswith(".whl")] if not wheels: raise FileNotFoundError(f"no wheels in {WHEELHOUSE}") RUNTIME_DIR.mkdir(parents=True, exist_ok=True) subprocess.run([sys.executable,"-m","pip","install","--no-index","--no-deps","--upgrade", "--target",str(RUNTIME_DIR)] + wheels, check=True, timeout=300) sys.path.insert(0, str(RUNTIME_DIR)); importlib.invalidate_caches() try: print("transformers:", importlib.metadata.version("transformers"), flush=True) except Exception: pass except Exception as e: emergency(f"wheel install failed: {e}"); raise import re, json, time import pandas as pd, torch from transformers import AutoTokenizer, AutoModelForCausalLM import ast as _ast, hashlib as _hash from fractions import Fraction as _Frac from collections import OrderedDict as _OD MODEL_ID="."; TIME_LIMIT=30*60; start=time.time() def write_csv(rows): import csv with open("submission.csv.tmp","w",newline="",encoding="utf-8") as f: w=csv.DictWriter(f,fieldnames=["id","pred","explanation"]); w.writeheader() for r in rows: w.writerow(r) os.replace("submission.csv.tmp","submission.csv") try: df = pd.read_csv("/tmp/data/test.csv", dtype=str).fillna("") write_csv([{"id":i,"pred":json.dumps([""]),"explanation":"placeholder"} for i in df["id"]]) tok = AutoTokenizer.from_pretrained(MODEL_ID, local_files_only=True) model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float16, device_map="auto", local_files_only=True).eval() print("loaded, quantized:", getattr(model.config,"quantization_config",None) is not None, flush=True) except Exception as e: emergency(f"load failed: {e}"); raise SYS=("You solve International Linguistics Olympiad problems about a language you have never seen. " "Everything you need is in the examples. Answer every numbered item, in order. " "Put each answer on its own line, with no numbering and no extra text.") def n_expected(query): items=re.findall(r"(?m)^\s*(\d+)\s*[.\)]", query) if items: return len(items) rng=re.search(r"\(\s*(\d+)\s*[-–—]\s*(\d+)\s*\)", query) if rng: lo,hi=int(rng.group(1)),int(rng.group(2)) if 0n: ans=ans[:n] if not ans: ans=[""] expl=re.sub(r"\s+"," ",text[:300]).strip() or "derived from the examples" rows.append({"id":r["id"],"pred":json.dumps(ans,ensure_ascii=False),"explanation":expl}) except Exception as e: n=n_expected(r["query"]) or 1 rows.append({"id":r["id"],"pred":json.dumps([""]*n,ensure_ascii=False),"explanation":"fallback"}) print("row error",r["id"],e,flush=True) done.add(r["id"]); write_csv(rows) print(f"{len(rows)}/{len(df)} t={time.time()-start:.0f}s",flush=True) if time.time()-start>TIME_LIMIT-60: print("time up, stopping",flush=True); break for _,r in df.iterrows(): if r["id"] in done: continue n=n_expected(r["query"]) or 1 rows.append({"id":r["id"],"pred":json.dumps([""]*n,ensure_ascii=False),"explanation":"fallback"}) # ========================================================================== # PASS 2: Grammar induction consensus. # Appended after the proven 0.121 baseline completes. The submission.csv # already has valid answers at this point. Pass 2 only IMPROVES rows where # two independent grammar inductions agree exactly -- never empties them. # ========================================================================== _P2_INDUCTION_SYS = ( "You study an International Linguistics Olympiad problem. " "From the examples only, write a RULE SHEET: with 3-8 bullet points " "covering the grammar: word meanings, word order, morphology, numeral " "composition, and exact-form constraints. " "Do not answer the queries. List only rules verifiable from examples." ) _P2_APPLICATION_SYS = ( "Apply the rule sheet to the IOL queries. " "Use only the rule sheet and examples. " "Follow the exact output format. No alternatives, no explanations." ) _P2_SOFT_DEADLINE = 1620 _P2_RULE_CAP = 90 _P2_APPLY_CAP = 110 _P2_MIN_T = 25 _P2_SAMPLES = 2 _THINK_RE2 = re.compile(r"=2 and len(set(labels))==len(labels) and numbered==len(labels): return tuple(labels) return None def _p2_safe_eval(expr): expr = (expr.replace("×","*").replace("÷","/").replace("·","*") .replace("\u2212","-").replace("^","**").strip()) if not expr or len(expr)>120: return None try: tree = _ast.parse(expr, mode="eval") except (SyntaxError,ValueError): return None def ev(n): if isinstance(n,_ast.Expression): return ev(n.body) if isinstance(n,_ast.Constant) and isinstance(n.value,(int,float)) and not isinstance(n.value,bool): return _Frac(str(n.value)) if isinstance(n,_ast.UnaryOp) and isinstance(n.op,(_ast.UAdd,_ast.USub)): v=ev(n.operand); return -v if isinstance(n.op,_ast.USub) else v if isinstance(n,_ast.BinOp): l,r=ev(n.left),ev(n.right); op=n.op if isinstance(op,_ast.Add): return l+r if isinstance(op,_ast.Sub): return l-r if isinstance(op,_ast.Mult): return l*r if isinstance(op,_ast.Div): if r==0: raise ValueError return l/r if isinstance(op,_ast.Pow): if r.denominator!=1 or not 0<=r.numerator<=10: raise ValueError return l**r.numerator raise ValueError try: v=ev(tree) return None if abs(v.numerator)>10**15 else v except Exception: return None def _p2_verify_arith(answers, checks): if len(answers)!=len(checks): return False for ans,chk in zip(answers,checks): if chk.count("=")!=1: return False l,r=chk.split("=",1) lv,rv=_p2_safe_eval(l),_p2_safe_eval(r) if lv is None or rv is None or lv!=rv: return False av=_p2_safe_eval(ans) if av is None or av!=rv: return False return True def _p2_invalid(s): return bool(s.startswith("```") or _THINK_RE2.search(s) or _FINAL_RE2.match(s) or _CHKRE2.match(s) or _FMT2.match(s) or _JUNK2.match(s) or (s.startswith("<") and s.endswith(">")) or s.casefold() in {"n/a","unknown","?","-"}) def _p2_parse_block(text, n): if not text or n<=0 or _THINK_RE2.search(text) or "```" in text: return None lines=[l.strip() for l in text.splitlines()] marks=[i for i,l in enumerate(lines) if _FINAL_RE2.match(l)] if len(marks)!=1: return None m=marks[0] if any(l for l in lines[:m]): return None answers=[l for l in lines[m+1:] if l] if len(answers)!=n or any(_p2_invalid(a) for a in answers): return None chk_marks=[i for i,l in enumerate(lines[:m]) if _CHKRE2.match(l)] checks=None if chk_marks: if len(chk_marks)!=1 or any(lines[:chk_marks[0]]): return None checks=[l for l in lines[chk_marks[0]+1:m] if l] if len(checks)!=n: return None return (answers, checks) def _p2_parse_group(text, counts): if not text: return {} lines=[l.strip() for l in text.splitlines()] results={}; seen=set(); dups=set() for si,line in enumerate(lines): bm=_BEGIN_RE2.match(line) if not bm: continue ordinal=int(bm.group(1).split("_",1)[1])-1 if not 0<=ordinal0 else 0.0 def _p2_seed(grow, idx): payload=json.dumps([{"c":_p2_field(r,"context")[:200]} for r in grow[:1]],sort_keys=True,separators=(",",":")) h=_hash.sha256(f"{idx}:{payload}".encode()).digest() return int.from_bytes(h[:8],"big")%(2**31) def _p2_guidance(grow): fams={_p2_classify(r) for r in grow} parts=[] if "match_letters" in fams: parts.append("Matching: complete one-to-one correspondence.") if "text_to_num" in fams: parts.append("Numbers: composition rules, verify arithmetic.") if "translation" in fams: parts.append("Translation: vocabulary, word order, morphology.") if "fill_blanks" in fams: parts.append("Blanks: exact morphological transformation.") if "num_to_text" in fams: parts.append("Numeral construction: base, order, word forms.") return " ".join(parts) def _p2_induction_msgs(grow): ctx=_p2_field(grow[0],"context") hints=[f"[{i}; {_p2_classify(r)}]\n{_p2_field(r,'query')}" for i,r in enumerate(grow,1)] return [{"role":"system","content":_P2_INDUCTION_SYS}, {"role":"user","content":(f"Focus: {_p2_guidance(grow)}\n\nEXAMPLES:\n{ctx}\n\n" f"QUERIES (do not answer):\n"+"\n\n".join(hints))}] def _p2_output_spec(r): fam=_p2_classify(r) if fam=="match_letters": return "One option label per line. Complete bijection." if fam=="text_to_num": return "Digits only. Optional ARITHMETIC CHECKS: block before FINAL ANSWERS:." if fam=="fill_blanks": return "One filled form per blank." if fam=="num_to_text": return "One written numeral per item." return "One translation per item. Exact surface form." def _p2_app_msgs(grow, rules, counts): ctx=_p2_field(grow[0],"context") blocks=[f"ROW_{i} ({_p2_classify(r)}, {n} answers):\n{_p2_field(r,'query')}\nFormat: {_p2_output_spec(r)}" for i,(r,n) in enumerate(zip(grow,counts),1)] shapes=[f"BEGIN ROW_{i}\nFINAL ANSWERS:\n<{n} lines>\nEND ROW_{i}" for i,n in enumerate(counts,1)] return [{"role":"system","content":_P2_APPLICATION_SYS}, {"role":"user","content":(f"EXAMPLES:\n{ctx}\n\nRULE SHEET:\n{rules}\n\n" f"QUERIES:\n"+"\n\n".join(blocks)+"\n\nExact structure:\n"+"\n\n".join(shapes))}] def _p2_clean_rules(text): text=(text or "").strip() if not text or _THINK_RE2.search(text) or "```" in text: return None if re.search(r"FINAL\s+ANSWERS\s*:",text,re.I): return None lines=[l.rstrip() for l in text.splitlines()] if lines and re.match(r"^RULE\s+SHEET\s*:?$",lines[0].strip(),re.I): lines=lines[1:] text="\n".join(lines).strip() return text if len(text)>=15 else None # Group rows by shared context, sort by priority _p2_grps=_OD() for _p2_ri,(_,_p2_r) in enumerate(df.iterrows()): _p2_grps.setdefault(_p2_field(_p2_r,"context"),[]).append(_p2_ri) def _p2_prio(g): fams={_p2_classify(df.iloc[i]) for i in g} if fams&{"match_letters","text_to_num"}: fp=0 elif fams&{"fill_blanks","num_to_text"}: fp=1 else: fp=2 return fp,-len(g),g[0] _p2_all=sorted(_p2_grps.values(),key=_p2_prio) _p2_rem=len(_p2_all)*_P2_SAMPLES*2 _p2_avail=_P2_SOFT_DEADLINE-_p2_elapsed() _p2_max=int(_p2_avail/(_P2_SAMPLES*2*_P2_MIN_T)) if _p2_avail>0 else 0 _p2_planned=_p2_all[:_p2_max] print(f"pass2: {len(_p2_planned)}/{len(_p2_all)} groups t={_p2_elapsed():.0f}s",flush=True) for _p2_gn,_p2_group in enumerate(_p2_planned,1): if _p2_elapsed()>_P2_SOFT_DEADLINE: break _p2_grow=[df.iloc[i] for i in _p2_group] _p2_counts=[_p2_n(r) or len(json.loads(rows[i]["pred"])) for i,r in zip(_p2_group,_p2_grow)] if any(c<=0 for c in _p2_counts): _p2_rem-=_P2_SAMPLES*2; continue _p2_cands=[[] for _ in _p2_group] for _p2_s in range(_P2_SAMPLES): _p2_rt=_p2_budget(_p2_rem,_P2_RULE_CAP); _p2_rem-=1 if _p2_rt<_P2_MIN_T: _p2_rem=0; break _p2_rules=None try: _p2_rules=_p2_clean_rules(_p2_sampled(_p2_induction_msgs(_p2_grow),640,_p2_rt,_p2_seed(_p2_grow,_p2_s)) or "") except Exception as _e: print(f"rule g{_p2_gn} s{_p2_s}: {_e}",flush=True) _p2_at=_p2_budget(_p2_rem,_P2_APPLY_CAP); _p2_rem-=1 _p2_val={} if _p2_rules and _p2_at>=_P2_MIN_T: try: _p2_raw=_p2_greedy(_p2_app_msgs(_p2_grow,_p2_rules,_p2_counts),1536,_p2_at) _p2_val=_p2_validate(_p2_grow,_p2_parse_group(_p2_raw or "",_p2_counts),_p2_counts) except Exception as _e: print(f"apply g{_p2_gn} s{_p2_s}: {_e}",flush=True) for _p2_j in range(len(_p2_group)): _p2_cands[_p2_j].append(_p2_val.get(_p2_j)) _p2_chg=0 for _p2_j,_p2_idx in enumerate(_p2_group): _p2_bl=json.loads(rows[_p2_idx]["pred"]) _p2_fin=_p2_consensus(_p2_grow[_p2_j],_p2_bl,_p2_counts[_p2_j],_p2_cands[_p2_j]) if _p2_fin!=_p2_bl: _p2_chg+=1; rows[_p2_idx]["pred"]=json.dumps(_p2_fin,ensure_ascii=False) write_csv(rows) print(f"pass2 g{_p2_gn}/{len(_p2_planned)} chg={_p2_chg} t={_p2_elapsed():.0f}s",flush=True) if _p2_rem<=0: break write_csv(rows); print("DONE",flush=True) except Exception as e: emergency(f"main loop: {e}"); print("FATAL",e,flush=True)