24 lines
1.3 KiB
Python
24 lines
1.3 KiB
Python
#!/usr/bin/env python
|
|
# Inference via the merged fp16 model (Transformers). pip install transformers accelerate torch
|
|
import argparse, json, os, sys
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE)
|
|
PROMPT = open(os.path.join(ROOT, "prompt.txt"), encoding="utf-8").read()
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--text"); ap.add_argument("--file"); ap.add_argument("--model", default=ROOT)
|
|
a = ap.parse_args()
|
|
text = open(a.file, encoding="utf-8").read() if a.file else (a.text or sys.stdin.read())
|
|
tok = AutoTokenizer.from_pretrained(a.model)
|
|
model = AutoModelForCausalLM.from_pretrained(a.model, torch_dtype="auto", device_map="auto")
|
|
ids = tok.apply_chat_template([{"role": "user", "content": PROMPT + text[:4000]}],
|
|
return_tensors="pt", add_generation_prompt=True).to(model.device)
|
|
out = model.generate(ids, max_new_tokens=768, do_sample=False, pad_token_id=tok.eos_token_id)
|
|
raw = tok.decode(out[0, ids.shape[1]:], skip_special_tokens=True)
|
|
try:
|
|
print(json.dumps(json.loads(raw[raw.find("{"):raw.rfind("}") + 1]), indent=2, ensure_ascii=False))
|
|
except Exception:
|
|
print(raw)
|
|
if __name__ == "__main__":
|
|
main()
|