dante-2b-ita-instruct/tokenizer_config.json

{
  "added_tokens_decoder": {
    "0": {
      "content": "<|begin_of_text|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "1": {
      "content": "<|end_of_text|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "2": {
      "content": "<|pad|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "3": {
      "content": "<|unk|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "4": {
      "content": "<|sep|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "5": {
      "content": "<|mask|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "6": {
      "content": "<|start_header|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "7": {
      "content": "<|end_header|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "8": {
      "content": "<|eot|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "9": {
      "content": "<|system|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "10": {
      "content": "<|user|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "11": {
      "content": "<|assistant|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "12": {
      "content": "<think>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "13": {
      "content": "</think>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "14": {
      "content": "<|code|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "15": {
      "content": "<|/code|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "16": {
      "content": "<|tool_call|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "17": {
      "content": "<|tool_result|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "18": {
      "content": "<|/tool_call|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "19": {
      "content": "<|/tool_result|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "20": {
      "content": "<|expert_0|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "21": {
      "content": "<|expert_1|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "22": {
      "content": "<|expert_2|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "23": {
      "content": "<|expert_3|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "24": {
      "content": "<|expert_4|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "25": {
      "content": "<|expert_5|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "26": {
      "content": "<|expert_6|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "27": {
      "content": "<|expert_7|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "28": {
      "content": "<|expert_8|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "29": {
      "content": "<|expert_9|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "30": {
      "content": "<|expert_10|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "31": {
      "content": "<|expert_11|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "32": {
      "content": "<|expert_12|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "33": {
      "content": "<|expert_13|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "34": {
      "content": "<|expert_14|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    },
    "35": {
      "content": "<|expert_15|>",
      "lstrip": false,
      "normalized": false,
      "rstrip": false,
      "single_word": true,
      "special": true
    }
  },
  "additional_special_tokens": [
    "<|start_header|>",
    "<|end_header|>",
    "<|eot|>",
    "<|system|>",
    "<|user|>",
    "<|assistant|>",
    "<think>",
    "</think>",
    "<|code|>",
    "<|/code|>",
    "<|tool_call|>",
    "<|tool_result|>",
    "<|/tool_call|>",
    "<|/tool_result|>",
    "<|expert_0|>",
    "<|expert_1|>",
    "<|expert_2|>",
    "<|expert_3|>",
    "<|expert_4|>",
    "<|expert_5|>",
    "<|expert_6|>",
    "<|expert_7|>",
    "<|expert_8|>",
    "<|expert_9|>",
    "<|expert_10|>",
    "<|expert_11|>",
    "<|expert_12|>",
    "<|expert_13|>",
    "<|expert_14|>",
    "<|expert_15|>"
  ],
  "bos_token": "<|begin_of_text|>",
  "eos_token": "<|end_of_text|>",
  "pad_token": "<|pad|>",
  "unk_token": "<|unk|>",
  "sep_token": "<|sep|>",
  "mask_token": "<|mask|>",
  "clean_up_tokenization_spaces": false,
  "model_max_length": 4096,
  "tokenizer_class": "PreTrainedTokenizerFast"
}