203 lines
4.3 KiB
JSON
203 lines
4.3 KiB
JSON
{
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<pad>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"3": {
|
|
"content": "<|endoftext|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"603": {
|
|
"content": "--",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64000": {
|
|
"content": "free_writing",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64001": {
|
|
"content": "matching_pairs",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64002": {
|
|
"content": "reorder_tokens",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64003": {
|
|
"content": "fill_in_the_blanks",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64004": {
|
|
"content": "general_vocab",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64005": {
|
|
"content": "morphology_focus",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64006": {
|
|
"content": "synonym_focus",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64007": {
|
|
"content": "antonym_focus",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64008": {
|
|
"content": "definition_only",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64009": {
|
|
"content": "[LESSON]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64010": {
|
|
"content": "[PRACTICE_TEXT]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64011": {
|
|
"content": "[EXERCISE_VARIATIONS]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64012": {
|
|
"content": "[A1]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64013": {
|
|
"content": "[A2]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64014": {
|
|
"content": "[B1]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"64015": {
|
|
"content": "[B2]",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"additional_special_tokens": [
|
|
"--",
|
|
"free_writing",
|
|
"matching_pairs",
|
|
"reorder_tokens",
|
|
"fill_in_the_blanks",
|
|
"general_vocab",
|
|
"morphology_focus",
|
|
"synonym_focus",
|
|
"antonym_focus",
|
|
"definition_only",
|
|
"[LESSON]",
|
|
"[PRACTICE_TEXT]",
|
|
"[EXERCISE_VARIATIONS]",
|
|
"[A1]",
|
|
"[A2]",
|
|
"[B1]",
|
|
"[B2]"
|
|
],
|
|
"bos_token": "<s>",
|
|
"clean_up_tokenization_spaces": false,
|
|
"do_lower_case": false,
|
|
"eos_token": "<|endoftext|>",
|
|
"extra_special_tokens": {},
|
|
"keep_accents": false,
|
|
"model_max_length": 4096,
|
|
"pad_token": "<pad>",
|
|
"remove_space": false,
|
|
"sp_model_kwargs": {},
|
|
"tokenizer_class": "GPTSw3Tokenizer",
|
|
"unk_token": "<unk>"
|
|
} |