37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
|
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||
|
|
import torch
|
||
|
|
|
||
|
|
|
||
|
|
class EndpointHandler:
|
||
|
|
def __init__(self, path=""):
|
||
|
|
self.tokenizer = AutoTokenizer.from_pretrained(path)
|
||
|
|
self.model = AutoModelForCausalLM.from_pretrained(
|
||
|
|
path,
|
||
|
|
torch_dtype=torch.float16,
|
||
|
|
device_map="auto"
|
||
|
|
)
|
||
|
|
|
||
|
|
def __call__(self, data):
|
||
|
|
messages = data.get("messages", [])
|
||
|
|
max_tokens = data.get("max_tokens", 2200)
|
||
|
|
|
||
|
|
text = self.tokenizer.apply_chat_template(
|
||
|
|
messages,
|
||
|
|
tokenize=False,
|
||
|
|
add_generation_prompt=True
|
||
|
|
)
|
||
|
|
inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
outputs = self.model.generate(
|
||
|
|
**inputs,
|
||
|
|
max_new_tokens=max_tokens,
|
||
|
|
do_sample=False
|
||
|
|
)
|
||
|
|
|
||
|
|
response = self.tokenizer.decode(
|
||
|
|
outputs[0][inputs["input_ids"].shape[1]:],
|
||
|
|
skip_special_tokens=True
|
||
|
|
)
|
||
|
|
return {"generated_text": response}
|