9466301768
- 3629499b7f0f5d9b7290ee866e480597430e728b5a87749e3f081628331fb108 (ef3eecf8ddfb0162c2a193dc2994c07dfd606c3e) - 3778ffe34626936b804e0b940cb8fb912db8599efedfbb248f87cd6511e2731b (97a423976d2218d6fe6c5a7bd6009b48cccdfbe0) - 52784d9778992d091d82d05e28031cb2405e52939b9ce4f9fac3d4c721f8ef32 (7dc0b68baaa8870615ec6ed99c53eef0fa481012) - 664bb4b330c67987c5d81579c56c62758979c0d4ab99315f6e241d2c89ff0388 (6b2044e4809bd11cc1319dfc7f7d809395fe8913) - e84d036c237c60c75bd8c1d5f1972443c9515980f827b396e18468b5eb1aaf3c (684ae7cab7c31853a134b0e25780a1fe52853325) - f1945ae592e762ecefdd284d8970a510817a9fc5d444ba694757faa3f220e791 (a4a4294e8f50f8f32f2ad62181bce43f7b616dbc)
132 lines
3.1 KiB
JSON
132 lines
3.1 KiB
JSON
{
|
|
"add_bos_token": false,
|
|
"add_eos_token": false,
|
|
"add_prefix_space": null,
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"32000": {
|
|
"content": "<|endoftext|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32001": {
|
|
"content": "<|assistant|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32002": {
|
|
"content": "<|placeholder1|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32003": {
|
|
"content": "<|placeholder2|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32004": {
|
|
"content": "<|placeholder3|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32005": {
|
|
"content": "<|placeholder4|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32006": {
|
|
"content": "<|system|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32007": {
|
|
"content": "<|end|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32008": {
|
|
"content": "<|placeholder5|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32009": {
|
|
"content": "<|placeholder6|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32010": {
|
|
"content": "<|user|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": true,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"bos_token": "<s>",
|
|
"chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "<|endoftext|>",
|
|
"legacy": false,
|
|
"model_max_length": 131072,
|
|
"pad_token": "<|endoftext|>",
|
|
"padding_side": "left",
|
|
"sp_model_kwargs": {},
|
|
"tokenizer_class": "LlamaTokenizer",
|
|
"unk_token": "<unk>",
|
|
"use_default_system_prompt": false
|
|
}
|