{
  "name": "veyra-tokenizer-8k",
  "vocab_size": 8192,
  "tokenizer_type": "BPE ByteLevel",
  "language": "English-focused",
  "chat_format": "ChatML",
  "training_corpus": {
    "total_texts": 250000,
    "mix": {
      "fineweb_edu": 0.7,
      "synthetic_edu": 0.1,
      "chatml": 0.08,
      "markdown_docs": 0.05,
      "json_tools": 0.04,
      "python_fim": 0.03
    },
    "fineweb_edu": "HuggingFaceFW/fineweb-edu sample-10BT streaming",
    "synthetic_sources": [
      "generated educational prose templates",
      "generated ChatML examples",
      "generated markdown/docs examples",
      "generated JSON/tool/context examples",
      "generated Python/FIM examples"
    ]
  },
  "special_tokens": [
    "<|bos|>",
    "<|eos|>",
    "<|pad|>",
    "<|unk|>",
    "<|im_start|>",
    "<|im_end|>",
    "<|tool_call|>",
    "<|tool_result|>",
    "<|context|>",
    "<|reasoning|>",
    "<|end_reasoning|>",
    "<|answer|>",
    "<|fim_prefix|>",
    "<|fim_middle|>",
    "<|fim_suffix|>",
    "<|reserved_0|>",
    "<|reserved_1|>",
    "<|reserved_2|>",
    "<|reserved_3|>",
    "<|reserved_4|>",
    "<|reserved_5|>",
    "<|reserved_6|>",
    "<|reserved_7|>",
    "<|reserved_8|>",
    "<|reserved_9|>"
  ],
  "notes": [
    "Designed for Veyra 30M-class attention-sparse decoder models.",
    "8k vocab including special tokens.",
    "Includes ChatML, tool/context, reasoning experiment, FIM, and reserved tokens."
  ]
}