1 line
2.9 MiB
JSON
1 line
2.9 MiB
JSON
|
{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 100256, "content": "<|extra_id_0|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100257, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100258, "content": "<|fim_prefix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100259, "content": "<|fim_middle|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100260, "content": "<|fim_suffix|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100261, "content": "|||PHONE_NUMBER|||", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100262, "content": "|||EMAIL_ADDRESS|||", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100263, "content": "|||IP_ADDRESS|||", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100264, "content": "<|im_start|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100265, "content": "<|im_end|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100266, "content": "<|extra_id_1|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100267, "content": "<|extra_id_2|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100268, "content": "<|extra_id_3|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100269, "content": "<|extra_id_4|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100270, "content": "<|extra_id_5|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100271, "content": "<|extra_id_6|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100272, "content": "<|extra_id_7|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100273, "content": "<|extra_id_8|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100274, "content": "<|extra_id_9|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100275, "content": "<|extra_id_10|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, {"id": 100276, "content": "<|endofprompt|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 100277, "content": "<|pad|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Removed", "invert": true}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": "", "end_of_word_suffix": "", "fuse_unk": false, "byte_fallback": false, "ignore_merges": false, "vocab": {"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27,
|