1 line
2.3 MiB
JSON
1 line
2.3 MiB
JSON
|
{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 0, "content": "<｜begin▁of▁sentence｜>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 1, "content": "<｜end▁of▁sentence｜>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 2, "content": "<｜▁pad▁｜>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64000, "content": "<|bos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64001, "content": "<|eos|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64002, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64003, "content": "<|unk|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64004, "content": "<|empty|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64005, "content": "<|very_short|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64006, "content": "<|short|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64007, "content": "<|long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64008, "content": "<|very_long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64009, "content": "<|tag_to_long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64010, "content": "<|long_to_tag|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64011, "content": "<|short_to_tag|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64012, "content": "<|short_to_long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64013, "content": "<|tag_to_short_to_long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64014, "content": "<|short_to_tag_to_long|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64015, "content": "<|short_to_long_to_tag|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64016, "content": "<|gen_meta|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64017, "content": "<|reserved_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64018, "content": "<|reserved_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64019, "content": "<|reserved_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64020, "content": "<|reserved_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64021, "content": "<|reserved_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64022, "content": "<|reserved_5|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64023, "content": "<|reserved_6|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64024, "content": "<|reserved_7|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 64025, "content": "<|reserved_8|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 6402
|