naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
HyperCLOVAX-SEED-Text-Instr…/tokenizer_config.json
ModelHub XC 784de26299 初始化项目，由ModelHub XC社区提供模型 
Model: naver-hyperclovax/HyperCLOVAX-SEED-Text-Instruct-0.5B
Source: Original Platform
2026-05-02 05:58:26 +08:00

503 lines
12 KiB
JSON
Raw Permalink Blame History

{
"add_prefix_space": false,
"added_tokens_decoder": {
"100256": {
"content": "<|_unuse_missing_100256|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100257": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100258": {
"content": "<|fim_prefix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100259": {
"content": "<|fim_middle|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100260": {
"content": "<|fim_suffix|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100261": {
"content": "<|_unuse_missing_100261|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100262": {
"content": "<|_unuse_missing_100262|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100263": {
"content": "<|_unuse_missing_100263|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100264": {
"content": "<|_unuse_missing_100264|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100265": {
"content": "<|_unuse_missing_100265|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100266": {
"content": "<|_unuse_missing_100266|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100267": {
"content": "<|_unuse_missing_100267|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100268": {
"content": "<|_unuse_missing_100268|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100269": {
"content": "<|_unuse_missing_100269|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100270": {
"content": "<|_unuse_missing_100270|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100271": {
"content": "<|_unuse_missing_100271|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100272": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100273": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100274": {
"content": "<|stop|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100275": {
"content": "<|endofturn|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100276": {
"content": "<|endofprompt|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110491": {
"content": "<repo_name>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110492": {
"content": "<file_sep>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110493": {
"content": "<issue_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110494": {
"content": "<issue_comment>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110495": {
"content": "<issue_closed>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110496": {
"content": "<jupyter_start>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110497": {
"content": "<jupyter_text>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110498": {
"content": "<jupyter_code>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110499": {
"content": "<jupyter_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110500": {
"content": "<jupyter_script>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110501": {
"content": "<empty_output>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110502": {
"content": "<code_to_intermediate>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110503": {
"content": "<intermediate_to_code>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110504": {
"content": "<pr>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110505": {
"content": "<pr_status>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110506": {
"content": "<pr_is_merged>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110507": {
"content": "<pr_base>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110508": {
"content": "<pr_file>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110509": {
"content": "<pr_base_code>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110510": {
"content": "<pr_diff>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110511": {
"content": "<pr_diff_hunk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110512": {
"content": "<pr_comment>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110513": {
"content": "<pr_event_id>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110514": {
"content": "<pr_review>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110515": {
"content": "<pr_review_state>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110516": {
"content": "<pr_review_comment>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110517": {
"content": "<pr_in_reply_to_review_id>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110518": {
"content": "<pr_in_reply_to_comment_id>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110519": {
"content": "<pr_diff_hunk_comment_line>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110520": {
"content": "<NAME>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110521": {
"content": "<EMAIL>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110522": {
"content": "<KEY>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"110523": {
"content": "<PASSWORD>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|endoftext|>",
"<|fim_prefix|>",
"<|fim_middle|>",
"<|fim_suffix|>",
"<|endofprompt|>",
"<|_unuse_missing_100256|>",
"<|_unuse_missing_100261|>",
"<|_unuse_missing_100262|>",
"<|_unuse_missing_100263|>",
"<|_unuse_missing_100264|>",
"<|_unuse_missing_100265|>",
"<|_unuse_missing_100266|>",
"<|_unuse_missing_100267|>",
"<|_unuse_missing_100268|>",
"<|_unuse_missing_100269|>",
"<|_unuse_missing_100270|>",
"<|_unuse_missing_100271|>",
"<|im_start|>",
"<|im_end|>",
"<|stop|>",
"<|endofturn|>",
"<repo_name>",
"<file_sep>",
"<issue_start>",
"<issue_comment>",
"<issue_closed>",
"<jupyter_start>",
"<jupyter_text>",
"<jupyter_code>",
"<jupyter_output>",
"<jupyter_script>",
"<empty_output>",
"<code_to_intermediate>",
"<intermediate_to_code>",
"<pr>",
"<pr_status>",
"<pr_is_merged>",
"<pr_base>",
"<pr_file>",
"<pr_base_code>",
"<pr_diff>",
"<pr_diff_hunk>",
"<pr_comment>",
"<pr_event_id>",
"<pr_review>",
"<pr_review_state>",
"<pr_review_comment>",
"<pr_in_reply_to_review_id>",
"<pr_in_reply_to_comment_id>",
"<pr_diff_hunk_comment_line>",
"<NAME>",
"<EMAIL>",
"<KEY>",
"<PASSWORD>"
],
"bos_token": "<|endoftext|>",
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": true,
"eos_token": "<|endofturn|>",
"extra_special_tokens": {},
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<|endoftext|>",
"tokenizer_class": "GPT2Tokenizer",
"unk_token": "<|endoftext|>"
}
Reference in New Issue View Git Blame Copy Permalink