qwen25-14b-awq-v2/script.py

"""IOL-AI 2026 submission entrypoint.

Runs on the platform's T4 (16 GB), offline, within a 30-minute limit. Reads the
hidden test set from /tmp/data/test.csv and writes submission.csv (id,pred) to
the working directory. Model weights ship inside this repo and load from ".".

Baseline model: Qwen/Qwen2.5-1.5B-Instruct. The score lever here is the harness
(task-aware prompts + robust parsing) in iol_harness.py, not the model size.
"""

import os
import time

# Must be set BEFORE importing transformers: no network at run time.
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")

import json

import pandas as pd
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

import iol_harness as H

MODEL_ID = "."  # weights are shipped in this repo (offline)
TEST_CSV = "/tmp/data/test.csv"
OUT_CSV = "submission.csv"

TIME_LIMIT_S = 30 * 60
SAFETY_S = 25 * 60  # stop generating past this; leaves buffer for a final gen+write
WRITE_EVERY = 3     # flush submission.csv every N problems (crash/timeout safety)
_START = time.monotonic()

# Optional per-repo harness config (reasoning-model repos ship one to turn on CoT).
CFG = {}
if os.path.exists("harness_cfg.json"):
    try:
        CFG = json.load(open("harness_cfg.json"))
    except Exception:
        CFG = {}
COT = bool(CFG.get("cot", False))
MAX_CAP = int(CFG.get("max_new_cap", 1024))
MAX_PASSES = int(CFG.get("sc_max_passes", 1))   # >1 turns on self-consistency
SC_TEMP = float(CFG.get("sc_temperature", 0.7))
FEWSHOT = bool(CFG.get("fewshot", False))        # prepend synthetic worked example


def elapsed():
    return time.monotonic() - _START


def load_model():
    tok = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
    model = AutoModelForCausalLM.from_pretrained(
        MODEL_ID,
        torch_dtype=torch.float16,  # T4 has no bfloat16
        device_map="auto",
        trust_remote_code=True,
    ).eval()
    if tok.pad_token_id is None:
        tok.pad_token = tok.eos_token
    return tok, model


def generate(tok, model, messages, max_new_tokens, sample=False, temperature=0.7):
    inputs = tok.apply_chat_template(
        messages,
        add_generation_prompt=True,
        return_tensors="pt",
    ).to(model.device)
    kwargs = dict(max_new_tokens=max_new_tokens, pad_token_id=tok.pad_token_id)
    if sample:
        kwargs.update(do_sample=True, temperature=temperature, top_p=0.95)
    else:
        kwargs.update(do_sample=False, num_beams=1)  # greedy
    with torch.no_grad():
        out = model.generate(inputs, **kwargs)
    return tok.decode(out[0][inputs.shape[-1] :], skip_special_tokens=True).strip()


def main():
    df = pd.read_csv(TEST_CSV, dtype=str).fillna("")
    total = len(df)
    print(f"loaded {total} problems from {TEST_CSV}", flush=True)

    # Pre-count items (no model needed) and write a COMPLETE all-blank
    # submission.csv up front. A valid file with every id then always exists —
    # even if we are killed during model load or a long generation — so a
    # timeout yields partial credit instead of total failure.
    ns = [
        H.count_items(r.get("context", ""), r.get("query", ""), r.get("task_type", ""))
        for _, r in df.iterrows()
    ]
    records = [
        {"id": df.iloc[i]["id"], "pred": json.dumps([""] * ns[i], ensure_ascii=False)}
        for i in range(total)
    ]

    def flush():
        pd.DataFrame(records, columns=["id", "pred"]).to_csv(OUT_CSV, index=False)

    flush()
    print(f"wrote blank {OUT_CSV} ({total} rows) at {elapsed():.0f}s", flush=True)

    tok, model = load_model()
    print(f"model loaded at {elapsed():.0f}s", flush=True)

    # Precompute per-problem prompt + token budget once.
    prompts, budgets = [], []
    for i in range(total):
        row = df.iloc[i]
        prompts.append(H.build_messages(
            row.get("context", ""), row.get("query", ""), row.get("task_type", ""), ns[i],
            cot=COT, fewshot=FEWSHOT))
        budgets.append(H.max_new_tokens_for(row.get("task_type", ""), ns[i], cot=COT, cap=MAX_CAP))

    # Self-consistency: pass 0 is greedy (a floor); passes 1+ are sampled. After
    # each pass we majority-vote across all passes and save — so we never do worse
    # than greedy, and extra time only helps. Time-budgeted to the 30-min limit.
    passes = []  # passes[k][i] = answer list for problem i in pass k
    est_pass = 0.0
    for k in range(max(1, MAX_PASSES)):
        if k > 0 and elapsed() + est_pass > SAFETY_S:
            print(f"[info] stopping before pass {k+1}: not enough time budget", flush=True)
            break
        t0 = elapsed()
        torch.manual_seed(1000 + k)  # reproducible sampling, diverse across passes
        this = []
        for i in range(total):
            if elapsed() > SAFETY_S:  # ran out mid-pass: blank the rest of this pass
                this += [[""] * ns[j] for j in range(i, total)]
                print(f"[warn] time budget hit mid-pass {k+1} at problem {i}", flush=True)
                break
            try:
                text = generate(tok, model, prompts[i], budgets[i],
                                sample=(k > 0), temperature=SC_TEMP)
                this.append(H.parse_answers(text, ns[i]))
            except Exception as exc:
                print(f"[warn] pass {k+1} id={df.iloc[i]['id']} failed: {exc!r}", flush=True)
                this.append([""] * ns[i])
        passes.append(this)
        est_pass = max(est_pass, elapsed() - t0)

        # vote across all passes so far and save
        for i in range(total):
            cands = [passes[p][i] for p in range(len(passes)) if i < len(passes[p])]
            records[i]["pred"] = json.dumps(H.majority_vote(cands, ns[i]), ensure_ascii=False)
        flush()
        print(f"pass {k+1}/{MAX_PASSES} done, voted+saved ({elapsed():.0f}s, ~{est_pass:.0f}s/pass)", flush=True)

    print(f"final {OUT_CSV} ({total} rows, {len(passes)} pass(es)) at {elapsed():.0f}s", flush=True)


if __name__ == "__main__":
    main()