Files
sglang/playground/load_tokenizer.py
2024-05-12 15:05:40 -07:00

9 lines
202 B
Python

import transformers
import code
#name = "meta-llama/Llama-2-7b-chat-hf"
name = "meta-llama/Meta-Llama-3-8B-Instruct"
t = transformers.AutoTokenizer.from_pretrained(name)
code.interact(local=locals())