初始化项目,由ModelHub XC社区提供模型
Model: timothywong731/tim-360m-instruct Source: Original Platform
This commit is contained in:
524
tokenizer_config.json
Normal file
524
tokenizer_config.json
Normal file
@@ -0,0 +1,524 @@
|
||||
{
|
||||
"tokenizer_class": "PreTrainedTokenizerFast",
|
||||
"model_max_length": 8192,
|
||||
"bos_token": "<|endoftext|>",
|
||||
"eos_token": "<|im_end|>",
|
||||
"pad_token": "<|pad|>",
|
||||
"unk_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<|endoftext|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"1": {
|
||||
"content": "<|im_start|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"2": {
|
||||
"content": "<|im_end|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"3": {
|
||||
"content": "<|pad|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"4": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"5": {
|
||||
"content": "<|fim_middle|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"6": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"7": {
|
||||
"content": "<|repo_name|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"8": {
|
||||
"content": "<|reserved_0|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"9": {
|
||||
"content": "<|reserved_1|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"10": {
|
||||
"content": "<|reserved_2|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"11": {
|
||||
"content": "<|reserved_3|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"12": {
|
||||
"content": "<|reserved_4|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"13": {
|
||||
"content": "<|reserved_5|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"14": {
|
||||
"content": "<|reserved_6|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"15": {
|
||||
"content": "<|reserved_7|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"16": {
|
||||
"content": "<|reserved_8|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"17": {
|
||||
"content": "<|reserved_9|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"18": {
|
||||
"content": "<|reserved_10|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"19": {
|
||||
"content": "<|reserved_11|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"20": {
|
||||
"content": "<|reserved_12|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"21": {
|
||||
"content": "<|reserved_13|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"22": {
|
||||
"content": "<|reserved_14|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"23": {
|
||||
"content": "<|reserved_15|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"24": {
|
||||
"content": "<|reserved_16|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"25": {
|
||||
"content": "<|reserved_17|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"26": {
|
||||
"content": "<|reserved_18|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"27": {
|
||||
"content": "<|reserved_19|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"28": {
|
||||
"content": "<|reserved_20|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"29": {
|
||||
"content": "<|reserved_21|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"30": {
|
||||
"content": "<|reserved_22|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"31": {
|
||||
"content": "<|reserved_23|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"32": {
|
||||
"content": "<|reserved_24|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"33": {
|
||||
"content": "<|reserved_25|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"34": {
|
||||
"content": "<|reserved_26|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"35": {
|
||||
"content": "<|reserved_27|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"36": {
|
||||
"content": "<|reserved_28|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"37": {
|
||||
"content": "<|reserved_29|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"38": {
|
||||
"content": "<|reserved_30|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"39": {
|
||||
"content": "<|reserved_31|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"40": {
|
||||
"content": "<|reserved_32|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"41": {
|
||||
"content": "<|reserved_33|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"42": {
|
||||
"content": "<|reserved_34|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"43": {
|
||||
"content": "<|reserved_35|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"44": {
|
||||
"content": "<|reserved_36|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"45": {
|
||||
"content": "<|reserved_37|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"46": {
|
||||
"content": "<|reserved_38|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"47": {
|
||||
"content": "<|reserved_39|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"48": {
|
||||
"content": "<|reserved_40|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"49": {
|
||||
"content": "<|reserved_41|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"50": {
|
||||
"content": "<|reserved_42|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"51": {
|
||||
"content": "<|reserved_43|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"52": {
|
||||
"content": "<|reserved_44|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"53": {
|
||||
"content": "<|reserved_45|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"54": {
|
||||
"content": "<|reserved_46|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"55": {
|
||||
"content": "<|reserved_47|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"56": {
|
||||
"content": "<|reserved_48|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"57": {
|
||||
"content": "<|reserved_49|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"58": {
|
||||
"content": "<|reserved_50|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"59": {
|
||||
"content": "<|reserved_51|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"60": {
|
||||
"content": "<|reserved_52|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"61": {
|
||||
"content": "<|reserved_53|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"62": {
|
||||
"content": "<|reserved_54|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
},
|
||||
"63": {
|
||||
"content": "<|reserved_55|>",
|
||||
"special": true,
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false
|
||||
}
|
||||
},
|
||||
"chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant\n'}}{% endif %}"
|
||||
}
|
||||
Reference in New Issue
Block a user