123 lines
2.7 KiB
JSON
123 lines
2.7 KiB
JSON
{
|
|
"add_bos_token": false,
|
|
"add_prefix_space": false,
|
|
"added_tokens_decoder": {
|
|
"50256": {
|
|
"content": "<|endoftext|>",
|
|
"lstrip": false,
|
|
"normalized": true,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50257": {
|
|
"content": "<|STOP|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50258": {
|
|
"content": "<|PAD|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50259": {
|
|
"content": "<|BEGIN_QUERY|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50260": {
|
|
"content": "<|BEGIN_ANALYSIS|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50261": {
|
|
"content": "<|END_ANALYSIS|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50262": {
|
|
"content": "<|BEGIN_RESPONSE|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50263": {
|
|
"content": "<|END_RESPONSE|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50264": {
|
|
"content": "<|BEGIN_SENTIMENT|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50265": {
|
|
"content": "<|END_SENTIMENT|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50266": {
|
|
"content": "<|BEGIN_CLASSIFICATION|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"50267": {
|
|
"content": "<|END_CLASSIFICATION|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"additional_special_tokens": [
|
|
"<|BEGIN_QUERY|>",
|
|
"<|BEGIN_QUERY|>",
|
|
"<|BEGIN_ANALYSIS|>",
|
|
"<|END_ANALYSIS|>",
|
|
"<|BEGIN_RESPONSE|>",
|
|
"<|END_RESPONSE|>",
|
|
"<|BEGIN_SENTIMENT|>",
|
|
"<|END_SENTIMENT|>",
|
|
"<|BEGIN_CLASSIFICATION|>",
|
|
"<|END_CLASSIFICATION|>"
|
|
],
|
|
"bos_token": "<|STOP|>",
|
|
"clean_up_tokenization_spaces": true,
|
|
"eos_token": "<|STOP|>",
|
|
"errors": "replace",
|
|
"model_max_length": 1024,
|
|
"pad_token": "<|PAD|>",
|
|
"tokenizer_class": "GPT2Tokenizer",
|
|
"unk_token": "<|endoftext|>"
|
|
}
|