mrfakename/smollm-135m-ipa-FAILED
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
smollm-135m-ipa-FAILED/tokenizer_config.json

769 lines
16 KiB
JSON
Raw Permalink Normal View History

初始化项目，由ModelHub XC社区提供模型 Model: mrfakename/smollm-135m-ipa-FAILED Source: Original Platform
2026-05-22 00:38:12 +08:00
{
"add_prefix_space": false,
"added_tokens_decoder": {
"0": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"3": {
"content": "<repo_name>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"4": {
"content": "<reponame>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"5": {
"content": "<file_sep>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"6": {
"content": "<filename>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"7": {
"content": "<gh_stars>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"8": {
"content": "<issue_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"9": {
"content": "<issue_comment>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"10": {
"content": "<issue_closed>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"11": {
"content": "<jupyter_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"12": {
"content": "<jupyter_text>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"13": {
"content": "<jupyter_code>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"14": {
"content": "<jupyter_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"15": {
"content": "<jupyter_script>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"16": {
"content": "<empty_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"17": {
"content": "!",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"24": {
"content": "(",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"25": {
"content": ")",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"28": {
"content": ",",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"30": {
"content": ".",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"33": {
"content": "1",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"42": {
"content": ":",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"43": {
"content": ";",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"47": {
"content": "?",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"75": {
"content": "[",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"77": {
"content": "]",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"81": {
"content": "a",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"82": {
"content": "b",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"84": {
"content": "d",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"85": {
"content": "e",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"86": {
"content": "f",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"88": {
"content": "h",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"89": {
"content": "i",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"90": {
"content": "j",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"91": {
"content": "k",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"92": {
"content": "l",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"93": {
"content": "m",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"94": {
"content": "n",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"95": {
"content": "o",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"96": {
"content": "p",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"98": {
"content": "r",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"99": {
"content": "s",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"100": {
"content": "t",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"101": {
"content": "u",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"102": {
"content": "v",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"103": {
"content": "w",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"104": {
"content": "x",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"106": {
"content": "z",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"111": {
"content": "¡",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"121": {
"content": "«",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"136": {
"content": "»",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"177": {
"content": "æ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"178": {
"content": "ç",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"187": {
"content": "ð",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49152": {
"content": "\t",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49153": {
"content": " ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49154": {
"content": "ŋ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49155": {
"content": "ɐ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49156": {
"content": "ɑ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49157": {
"content": "ɔ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49158": {
"content": "ə",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49159": {
"content": "ɚ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49160": {
"content": "ɛ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49161": {
"content": "ɜ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49162": {
"content": "ɡ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49163": {
"content": "ɪ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49164": {
"content": "ɬ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49165": {
"content": "ɹ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49166": {
"content": "ɾ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49167": {
"content": "ʃ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49168": {
"content": "ʊ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49169": {
"content": "ʌ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49170": {
"content": "ʒ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49171": {
"content": "ʔ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49172": {
"content": "ʲ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49173": {
"content": "ˈ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49174": {
"content": "ˌ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49175": {
"content": "ː",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49176": {
"content": "̃",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49177": {
"content": "̩",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49178": {
"content": "θ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49179": {
"content": "ᵻ",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49180": {
"content": "—",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49181": {
"content": "“",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49182": {
"content": "”",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49183": {
"content": "…",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49184": {
"content": "<|text_start|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49185": {
"content": "<|text_end|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49186": {
"content": "<|ipa_start|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"49187": {
"content": "<|ipa_end|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"additional_special_tokens": [
"<|endoftext|>",
"<|im_start|>",
"<|im_end|>",
"<repo_name>",
"<reponame>",
"<file_sep>",
"<filename>",
"<gh_stars>",
"<issue_start>",
"<issue_comment>",
"<issue_closed>",
"<jupyter_start>",
"<jupyter_text>",
"<jupyter_code>",
"<jupyter_output>",
"<jupyter_script>",
"<empty_output>"
],
"bos_token": "<|endoftext|>",
"clean_up_tokenization_spaces": false,
"eos_token": "<|endoftext|>",
"extra_special_tokens": {},
"model_max_length": 8192,
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>",
"vocab_size": 49152
}
Reference in New Issue Copy Permalink