37 lines
1.3 KiB
Python
37 lines
1.3 KiB
Python
|
import gradio as gr
|
||
|
|
import spaces # Required for ZeroGPU hardware
|
||
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||
|
|
import torch
|
||
|
|
|
||
|
|
# Load your deployed model
|
||
|
|
model_id = "Zeesnal786/llama3-pakistani-fintech-3b"
|
||
|
|
|
||
|
|
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||
|
|
model = AutoModelForCausalLM.from_pretrained(
|
||
|
|
model_id,
|
||
|
|
device_map="auto",
|
||
|
|
torch_dtype=torch.bfloat16
|
||
|
|
)
|
||
|
|
|
||
|
|
# Decorate the function so ZeroGPU knows to allocate hardware here
|
||
|
|
@spaces.GPU
|
||
|
|
def chat_function(question):
|
||
|
|
messages = [{"role": "user", "content": question}]
|
||
|
|
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||
|
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
||
|
|
|
||
|
|
outputs = model.generate(**inputs, max_new_tokens=256)
|
||
|
|
answer = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
||
|
|
return answer
|
||
|
|
|
||
|
|
# Create the Gradio interface
|
||
|
|
demo = gr.Interface(
|
||
|
|
fn=chat_function,
|
||
|
|
inputs=gr.Textbox(label="Your Banking Question", placeholder="e.g. How do I register on NayaPay?"),
|
||
|
|
outputs=gr.Textbox(label="Model Answer"),
|
||
|
|
title="Pakistani Fintech FAQ",
|
||
|
|
description="Ask me any banking question regarding easypaisa, JS Bank, MCB, or NayaPay."
|
||
|
|
)
|
||
|
|
|
||
|
|
# Disable SSR to prevent the asyncio ValueError on startup
|
||
|
|
demo.launch(ssr_mode=False)
|