39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
# infer_simple.py
|
|
import torch
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
from prompt_format import format_instruction_prompt
|
|
|
|
MODEL_ID = "ericoh929/qwen3-1.7b-lamini-qlora-instruction-tuned"
|
|
|
|
def main():
|
|
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
MODEL_ID,
|
|
trust_remote_code=True,
|
|
device_map="auto",
|
|
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
|
)
|
|
model.eval()
|
|
|
|
instruction = "Answer the question. Be concise."
|
|
inp = "If Tom has 3 apples and buys 4 more, how many apples does he have?"
|
|
|
|
prompt = format_instruction_prompt(instruction, inp)
|
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
|
|
|
with torch.inference_mode():
|
|
out = model.generate(
|
|
**inputs,
|
|
max_new_tokens=256,
|
|
do_sample=False,
|
|
pad_token_id=tokenizer.pad_token_id or tokenizer.eos_token_id,
|
|
eos_token_id=tokenizer.eos_token_id,
|
|
)
|
|
|
|
gen_ids = out[0][inputs["input_ids"].shape[1]:]
|
|
answer = tokenizer.decode(gen_ids, skip_special_tokens=True).strip()
|
|
|
|
print(answer)
|
|
|
|
if __name__ == "__main__":
|
|
main() |