30 lines
835 B
Python
30 lines
835 B
Python
from datasets import load_from_disk
|
|
from tokenizers import trainers, Tokenizer, normalizers, ByteLevelBPETokenizer
|
|
from transformers import AutoTokenizer
|
|
|
|
|
|
model_dir = "./"
|
|
|
|
# load dataset
|
|
dataset = load_from_disk("/researchdisk/training_dataset_full_deduplicated")
|
|
dataset = dataset["train"]
|
|
|
|
# Instantiate tokenizer
|
|
tokenizer = ByteLevelBPETokenizer()
|
|
def batch_iterator(batch_size=1000):
|
|
for i in range(0, len(dataset), batch_size):
|
|
yield dataset[i: i + batch_size]["text"]
|
|
|
|
# Customized training
|
|
tokenizer.train_from_iterator(batch_iterator(), vocab_size=50257, min_frequency=2, special_tokens=[
|
|
"<s>",
|
|
"<pad>",
|
|
"</s>",
|
|
"<unk>",
|
|
"<mask>",
|
|
])
|
|
|
|
# Save files to disk
|
|
tokenizer.save(f"{model_dir}/tokenizer.json")
|
|
tokenizer = AutoTokenizer.from_pretrained(model_dir)
|
|
tokenizer.save_pretrained(model_dir) |