58 lines
1.4 KiB
JSON
58 lines
1.4 KiB
JSON
|
{
|
||
|
|
"name": "veyra-tokenizer-8k",
|
||
|
|
"vocab_size": 8192,
|
||
|
|
"tokenizer_type": "BPE ByteLevel",
|
||
|
|
"language": "English-focused",
|
||
|
|
"chat_format": "ChatML",
|
||
|
|
"training_corpus": {
|
||
|
|
"total_texts": 250000,
|
||
|
|
"mix": {
|
||
|
|
"fineweb_edu": 0.7,
|
||
|
|
"synthetic_edu": 0.1,
|
||
|
|
"chatml": 0.08,
|
||
|
|
"markdown_docs": 0.05,
|
||
|
|
"json_tools": 0.04,
|
||
|
|
"python_fim": 0.03
|
||
|
|
},
|
||
|
|
"fineweb_edu": "HuggingFaceFW/fineweb-edu sample-10BT streaming",
|
||
|
|
"synthetic_sources": [
|
||
|
|
"generated educational prose templates",
|
||
|
|
"generated ChatML examples",
|
||
|
|
"generated markdown/docs examples",
|
||
|
|
"generated JSON/tool/context examples",
|
||
|
|
"generated Python/FIM examples"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"special_tokens": [
|
||
|
|
"<|bos|>",
|
||
|
|
"<|eos|>",
|
||
|
|
"<|pad|>",
|
||
|
|
"<|unk|>",
|
||
|
|
"<|im_start|>",
|
||
|
|
"<|im_end|>",
|
||
|
|
"<|tool_call|>",
|
||
|
|
"<|tool_result|>",
|
||
|
|
"<|context|>",
|
||
|
|
"<|reasoning|>",
|
||
|
|
"<|end_reasoning|>",
|
||
|
|
"<|answer|>",
|
||
|
|
"<|fim_prefix|>",
|
||
|
|
"<|fim_middle|>",
|
||
|
|
"<|fim_suffix|>",
|
||
|
|
"<|reserved_0|>",
|
||
|
|
"<|reserved_1|>",
|
||
|
|
"<|reserved_2|>",
|
||
|
|
"<|reserved_3|>",
|
||
|
|
"<|reserved_4|>",
|
||
|
|
"<|reserved_5|>",
|
||
|
|
"<|reserved_6|>",
|
||
|
|
"<|reserved_7|>",
|
||
|
|
"<|reserved_8|>",
|
||
|
|
"<|reserved_9|>"
|
||
|
|
],
|
||
|
|
"notes": [
|
||
|
|
"Designed for Veyra 30M-class attention-sparse decoder models.",
|
||
|
|
"8k vocab including special tokens.",
|
||
|
|
"Includes ChatML, tool/context, reasoning experiment, FIM, and reserved tokens."
|
||
|
|
]
|
||
|
|
}
|