64 lines
2.3 KiB
Python
64 lines
2.3 KiB
Python
import torch
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, HfArgumentParser, TrainingArguments, pipeline, logging, TextStreamer
|
|
from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training, get_peft_model
|
|
import os, torch, wandb, platform, warnings
|
|
from datasets import load_dataset
|
|
from trl import SFTTrainer
|
|
|
|
hf_token = '..........'
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained('./vistral-tokenizer')
|
|
bnb_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
bnb_4bit_use_double_quant=True,
|
|
)
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
'Viet-Mistral/Vistral-7B-Chat',
|
|
device_map="auto",
|
|
token=hf_token,
|
|
quantization_config=bnb_config,
|
|
)
|
|
ft_model = PeftModel.from_pretrained(model, CHECKPOINT_PATH)
|
|
|
|
#torch.backends.cuda.enable_mem_efficient_sdp(False)
|
|
#torch.backends.cuda.enable_flash_sdp(False)
|
|
|
|
system_prompt = "Bạn là một trợ lí Tiếng Việt nhiệt tình và trung thực. Hãy luôn trả lời một cách hữu ích nhất có thể, đồng thời giữ an toàn."
|
|
|
|
stop_tokens = [tokenizer.eos_token_id, tokenizer('<|im_end|>')['input_ids'].pop()]
|
|
|
|
def chat_test():
|
|
conversation = [{"role": "system", "content": system_prompt }]
|
|
while True:
|
|
human = input("Human: ")
|
|
if human.lower() == "reset":
|
|
conversation = [{"role": "system", "content": system_prompt }]
|
|
print("The chat history has been cleared!")
|
|
continue
|
|
|
|
if human.lower() == "exit":
|
|
break
|
|
|
|
conversation.append({"role": "user", "content": human })
|
|
formatted = tokenizer.apply_chat_template(conversation, tokenize=False) + "<|im_start|>assistant"
|
|
tok = tokenizer(formatted, return_tensors="pt").to(ft_model.device)
|
|
input_ids = tok['input_ids']
|
|
|
|
out_ids = ft_model.generate(
|
|
input_ids=input_ids,
|
|
attention_mask=tok['attention_mask'],
|
|
eos_token_id=stop_tokens,
|
|
max_new_tokens=50,
|
|
do_sample=True,
|
|
top_p=0.95,
|
|
top_k=40,
|
|
temperature=0.1,
|
|
repetition_penalty=1.05,
|
|
)
|
|
assistant = tokenizer.batch_decode(out_ids[:, input_ids.size(1): ], skip_special_tokens=True)[0].strip()
|
|
print("Assistant: ", assistant)
|
|
conversation.append({"role": "assistant", "content": assistant })
|
|
|
|
chat_test() |