#!/usr/bin/env python
# Inference via the merged fp16 model (Transformers).  pip install transformers accelerate torch
import argparse, json, os, sys
from transformers import AutoModelForCausalLM, AutoTokenizer
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
PROMPT = open(os.path.join(ROOT, "prompt.txt"), encoding="utf-8").read()
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--text"); ap.add_argument("--file"); ap.add_argument("--model", default=ROOT)
    a = ap.parse_args()
    text = open(a.file, encoding="utf-8").read() if a.file else (a.text or sys.stdin.read())
    tok = AutoTokenizer.from_pretrained(a.model)
    model = AutoModelForCausalLM.from_pretrained(a.model, torch_dtype="auto", device_map="auto")
    ids = tok.apply_chat_template([{"role": "user", "content": PROMPT + text[:4000]}],
                                  return_tensors="pt", add_generation_prompt=True).to(model.device)
    out = model.generate(ids, max_new_tokens=768, do_sample=False, pad_token_id=tok.eos_token_id)
    raw = tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)
    try:
        print(json.dumps(json.loads(raw[raw.find("{"):raw.rfind("}") + 1]), indent=2, ensure_ascii=False))
    except Exception:
        print(raw)
if __name__ == "__main__":
    main()