25 lines
636 B
Python
25 lines
636 B
Python
from datasets import load_dataset
|
|
from tokenizers import trainers, Tokenizer, normalizers, ByteLevelBPETokenizer
|
|
|
|
# load dataset
|
|
dataset = load_dataset("imthanhlv/binhvq_dedup", split="train")
|
|
|
|
# Instantiate tokenizer
|
|
tokenizer = ByteLevelBPETokenizer()
|
|
|
|
def batch_iterator(batch_size=1000):
|
|
for i in range(0, len(dataset), batch_size):
|
|
yield dataset[i: i + batch_size]["text"]
|
|
|
|
# Customized training
|
|
tokenizer.train_from_iterator(batch_iterator(), vocab_size=50257, min_frequency=2, special_tokens=[
|
|
"<s>",
|
|
"<pad>",
|
|
"</s>",
|
|
"<unk>",
|
|
"<mask>",
|
|
])
|
|
|
|
# Save files to disk
|
|
tokenizer.save("./tokenizer.json")
|