87 lines
2.6 KiB
JSON
87 lines
2.6 KiB
JSON
{
|
|
"add_bos_token": true,
|
|
"add_eos_token": false,
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32007": {
|
|
"content": "▁<PRE>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32008": {
|
|
"content": "▁<SUF>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32009": {
|
|
"content": "▁<MID>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"32010": {
|
|
"content": "▁<EOT>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
}
|
|
},
|
|
"additional_special_tokens": [
|
|
"▁<PRE>",
|
|
"▁<MID>",
|
|
"▁<SUF>",
|
|
"▁<EOT>"
|
|
],
|
|
"bos_token": "<s>",
|
|
"chat_template": "{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Please continue to complete the function. You are not allowed to modify the given code and do the completion only. Please return all completed function in a codeblock. Here is the given code to do completion:\n```python\n' + content + '\n' }}{% elif message['role'] == 'assistant' %}{{ '\n' + content + '\n```' + '\n' }}{% endif %}{% endfor %}",
|
|
"clean_up_tokenization_spaces": false,
|
|
"eos_token": "</s>",
|
|
"eot_token": "▁<EOT>",
|
|
"fill_token": "<FILL_ME>",
|
|
"legacy": null,
|
|
"middle_token": "▁<MID>",
|
|
"model_max_length": 1000000000000000019884624838656,
|
|
"pad_token": "</s>",
|
|
"padding_side": "right",
|
|
"prefix_token": "▁<PRE>",
|
|
"sp_model_kwargs": {},
|
|
"split_special_tokens": false,
|
|
"suffix_token": "▁<SUF>",
|
|
"tokenizer_class": "CodeLlamaTokenizer",
|
|
"unk_token": "<unk>",
|
|
"use_default_system_prompt": false
|
|
}
|