Files
OpenHermes-2.5-Strix-Philos…/special_tokens_map.json
Sayhan Yalvaçer 8c994d7735 Upload tokenizer
2024-02-18 08:40:15 +00:00

25 lines
444 B
JSON

{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<unk>",
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}