97 lines
1.9 KiB
JSON
97 lines
1.9 KiB
JSON
{
|
|
"additional_special_tokens": [
|
|
"<|im_start|>",
|
|
"<|im_end|>",
|
|
"<|special_0|>",
|
|
"<|special_1|>",
|
|
"<|special_2|>",
|
|
"<|special_3|>",
|
|
"<|special_4|>",
|
|
"<|special_5|>",
|
|
"<|special_6|>",
|
|
"<|special_7|>",
|
|
"<|special_8|>",
|
|
"<|special_9|>",
|
|
"<|special_10|>",
|
|
"<|special_11|>",
|
|
"<|special_12|>",
|
|
"<|special_13|>",
|
|
"<|special_14|>",
|
|
"<|special_15|>",
|
|
"<|special_16|>",
|
|
"<|special_17|>",
|
|
"<|special_18|>",
|
|
"<|special_19|>",
|
|
"<|special_20|>",
|
|
"<|special_21|>",
|
|
"<|special_22|>",
|
|
"<|special_23|>",
|
|
"<|special_24|>",
|
|
"<|special_25|>",
|
|
"<|special_26|>",
|
|
"<|special_27|>",
|
|
"<|special_28|>",
|
|
"<|special_29|>",
|
|
"<|special_30|>",
|
|
"<|special_31|>",
|
|
"<|special_32|>",
|
|
"<|special_33|>",
|
|
"<|special_34|>",
|
|
"<|special_35|>",
|
|
"<|special_36|>",
|
|
"<|special_37|>",
|
|
"<|special_38|>",
|
|
"<|special_39|>",
|
|
"<|special_40|>",
|
|
"<|special_41|>",
|
|
"<|special_42|>",
|
|
"<|special_43|>",
|
|
"<|special_44|>",
|
|
"<|special_45|>",
|
|
"<|special_46|>",
|
|
"<|special_47|>",
|
|
"<|special_48|>",
|
|
"<|special_49|>",
|
|
"<|special_50|>",
|
|
"<|special_51|>",
|
|
"<|special_52|>",
|
|
"<|special_53|>",
|
|
"<|special_54|>",
|
|
"<|special_55|>",
|
|
"<|special_56|>",
|
|
"<|special_57|>",
|
|
"<|special_58|>",
|
|
"<|special_59|>",
|
|
"<|special_60|>",
|
|
"<|special_61|>"
|
|
],
|
|
"bos_token": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"eos_token": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"pad_token": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"unk_token": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
}
|
|
}
|