2024-07-26 11:33:04 +00:00
{
"added_tokens_decoder" : {
"128000" : {
"content" : "<|begin_of_text|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128001" : {
"content" : "<|end_of_text|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128002" : {
"content" : "<|reserved_special_token_0|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128003" : {
"content" : "<|reserved_special_token_1|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128004" : {
"content" : "<|finetune_right_pad_id|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128005" : {
"content" : "<|reserved_special_token_2|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128006" : {
"content" : "<|start_header_id|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128007" : {
"content" : "<|end_header_id|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128008" : {
"content" : "<|eom_id|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128009" : {
"content" : "<|eot_id|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128010" : {
"content" : "<|python_tag|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128011" : {
"content" : "<|reserved_special_token_3|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128012" : {
"content" : "<|reserved_special_token_4|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128013" : {
"content" : "<|reserved_special_token_5|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128014" : {
"content" : "<|reserved_special_token_6|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128015" : {
"content" : "<|reserved_special_token_7|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128016" : {
"content" : "<|reserved_special_token_8|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128017" : {
"content" : "<|reserved_special_token_9|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128018" : {
"content" : "<|reserved_special_token_10|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128019" : {
"content" : "<|reserved_special_token_11|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128020" : {
"content" : "<|reserved_special_token_12|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128021" : {
"content" : "<|reserved_special_token_13|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128022" : {
"content" : "<|reserved_special_token_14|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128023" : {
"content" : "<|reserved_special_token_15|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128024" : {
"content" : "<|reserved_special_token_16|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128025" : {
"content" : "<|reserved_special_token_17|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128026" : {
"content" : "<|reserved_special_token_18|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128027" : {
"content" : "<|reserved_special_token_19|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128028" : {
"content" : "<|reserved_special_token_20|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128029" : {
"content" : "<|reserved_special_token_21|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128030" : {
"content" : "<|reserved_special_token_22|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128031" : {
"content" : "<|reserved_special_token_23|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128032" : {
"content" : "<|reserved_special_token_24|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128033" : {
"content" : "<|reserved_special_token_25|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128034" : {
"content" : "<|reserved_special_token_26|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128035" : {
"content" : "<|reserved_special_token_27|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128036" : {
"content" : "<|reserved_special_token_28|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128037" : {
"content" : "<|reserved_special_token_29|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128038" : {
"content" : "<|reserved_special_token_30|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128039" : {
"content" : "<|reserved_special_token_31|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128040" : {
"content" : "<|reserved_special_token_32|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128041" : {
"content" : "<|reserved_special_token_33|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128042" : {
"content" : "<|reserved_special_token_34|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128043" : {
"content" : "<|reserved_special_token_35|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128044" : {
"content" : "<|reserved_special_token_36|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128045" : {
"content" : "<|reserved_special_token_37|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128046" : {
"content" : "<|reserved_special_token_38|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128047" : {
"content" : "<|reserved_special_token_39|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128048" : {
"content" : "<|reserved_special_token_40|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128049" : {
"content" : "<|reserved_special_token_41|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128050" : {
"content" : "<|reserved_special_token_42|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128051" : {
"content" : "<|reserved_special_token_43|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128052" : {
"content" : "<|reserved_special_token_44|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128053" : {
"content" : "<|reserved_special_token_45|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128054" : {
"content" : "<|reserved_special_token_46|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128055" : {
"content" : "<|reserved_special_token_47|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128056" : {
"content" : "<|reserved_special_token_48|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128057" : {
"content" : "<|reserved_special_token_49|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128058" : {
"content" : "<|reserved_special_token_50|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128059" : {
"content" : "<|reserved_special_token_51|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128060" : {
"content" : "<|reserved_special_token_52|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128061" : {
"content" : "<|reserved_special_token_53|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128062" : {
"content" : "<|reserved_special_token_54|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128063" : {
"content" : "<|reserved_special_token_55|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128064" : {
"content" : "<|reserved_special_token_56|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128065" : {
"content" : "<|reserved_special_token_57|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128066" : {
"content" : "<|reserved_special_token_58|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128067" : {
"content" : "<|reserved_special_token_59|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128068" : {
"content" : "<|reserved_special_token_60|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128069" : {
"content" : "<|reserved_special_token_61|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128070" : {
"content" : "<|reserved_special_token_62|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128071" : {
"content" : "<|reserved_special_token_63|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128072" : {
"content" : "<|reserved_special_token_64|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128073" : {
"content" : "<|reserved_special_token_65|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128074" : {
"content" : "<|reserved_special_token_66|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128075" : {
"content" : "<|reserved_special_token_67|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128076" : {
"content" : "<|reserved_special_token_68|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128077" : {
"content" : "<|reserved_special_token_69|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128078" : {
"content" : "<|reserved_special_token_70|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128079" : {
"content" : "<|reserved_special_token_71|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128080" : {
"content" : "<|reserved_special_token_72|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128081" : {
"content" : "<|reserved_special_token_73|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128082" : {
"content" : "<|reserved_special_token_74|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128083" : {
"content" : "<|reserved_special_token_75|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128084" : {
"content" : "<|reserved_special_token_76|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128085" : {
"content" : "<|reserved_special_token_77|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128086" : {
"content" : "<|reserved_special_token_78|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128087" : {
"content" : "<|reserved_special_token_79|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128088" : {
"content" : "<|reserved_special_token_80|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128089" : {
"content" : "<|reserved_special_token_81|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128090" : {
"content" : "<|reserved_special_token_82|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128091" : {
"content" : "<|reserved_special_token_83|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128092" : {
"content" : "<|reserved_special_token_84|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128093" : {
"content" : "<|reserved_special_token_85|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128094" : {
"content" : "<|reserved_special_token_86|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128095" : {
"content" : "<|reserved_special_token_87|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128096" : {
"content" : "<|reserved_special_token_88|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128097" : {
"content" : "<|reserved_special_token_89|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128098" : {
"content" : "<|reserved_special_token_90|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128099" : {
"content" : "<|reserved_special_token_91|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128100" : {
"content" : "<|reserved_special_token_92|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128101" : {
"content" : "<|reserved_special_token_93|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128102" : {
"content" : "<|reserved_special_token_94|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128103" : {
"content" : "<|reserved_special_token_95|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128104" : {
"content" : "<|reserved_special_token_96|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128105" : {
"content" : "<|reserved_special_token_97|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128106" : {
"content" : "<|reserved_special_token_98|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128107" : {
"content" : "<|reserved_special_token_99|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128108" : {
"content" : "<|reserved_special_token_100|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128109" : {
"content" : "<|reserved_special_token_101|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128110" : {
"content" : "<|reserved_special_token_102|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128111" : {
"content" : "<|reserved_special_token_103|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128112" : {
"content" : "<|reserved_special_token_104|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128113" : {
"content" : "<|reserved_special_token_105|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128114" : {
"content" : "<|reserved_special_token_106|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128115" : {
"content" : "<|reserved_special_token_107|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128116" : {
"content" : "<|reserved_special_token_108|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128117" : {
"content" : "<|reserved_special_token_109|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128118" : {
"content" : "<|reserved_special_token_110|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128119" : {
"content" : "<|reserved_special_token_111|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128120" : {
"content" : "<|reserved_special_token_112|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128121" : {
"content" : "<|reserved_special_token_113|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128122" : {
"content" : "<|reserved_special_token_114|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128123" : {
"content" : "<|reserved_special_token_115|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128124" : {
"content" : "<|reserved_special_token_116|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128125" : {
"content" : "<|reserved_special_token_117|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128126" : {
"content" : "<|reserved_special_token_118|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128127" : {
"content" : "<|reserved_special_token_119|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128128" : {
"content" : "<|reserved_special_token_120|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128129" : {
"content" : "<|reserved_special_token_121|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128130" : {
"content" : "<|reserved_special_token_122|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128131" : {
"content" : "<|reserved_special_token_123|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128132" : {
"content" : "<|reserved_special_token_124|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128133" : {
"content" : "<|reserved_special_token_125|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128134" : {
"content" : "<|reserved_special_token_126|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128135" : {
"content" : "<|reserved_special_token_127|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128136" : {
"content" : "<|reserved_special_token_128|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128137" : {
"content" : "<|reserved_special_token_129|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128138" : {
"content" : "<|reserved_special_token_130|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128139" : {
"content" : "<|reserved_special_token_131|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128140" : {
"content" : "<|reserved_special_token_132|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128141" : {
"content" : "<|reserved_special_token_133|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128142" : {
"content" : "<|reserved_special_token_134|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128143" : {
"content" : "<|reserved_special_token_135|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128144" : {
"content" : "<|reserved_special_token_136|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128145" : {
"content" : "<|reserved_special_token_137|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128146" : {
"content" : "<|reserved_special_token_138|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128147" : {
"content" : "<|reserved_special_token_139|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128148" : {
"content" : "<|reserved_special_token_140|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128149" : {
"content" : "<|reserved_special_token_141|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128150" : {
"content" : "<|reserved_special_token_142|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128151" : {
"content" : "<|reserved_special_token_143|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128152" : {
"content" : "<|reserved_special_token_144|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128153" : {
"content" : "<|reserved_special_token_145|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128154" : {
"content" : "<|reserved_special_token_146|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128155" : {
"content" : "<|reserved_special_token_147|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128156" : {
"content" : "<|reserved_special_token_148|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128157" : {
"content" : "<|reserved_special_token_149|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128158" : {
"content" : "<|reserved_special_token_150|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128159" : {
"content" : "<|reserved_special_token_151|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128160" : {
"content" : "<|reserved_special_token_152|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128161" : {
"content" : "<|reserved_special_token_153|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128162" : {
"content" : "<|reserved_special_token_154|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128163" : {
"content" : "<|reserved_special_token_155|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128164" : {
"content" : "<|reserved_special_token_156|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128165" : {
"content" : "<|reserved_special_token_157|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128166" : {
"content" : "<|reserved_special_token_158|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128167" : {
"content" : "<|reserved_special_token_159|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128168" : {
"content" : "<|reserved_special_token_160|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128169" : {
"content" : "<|reserved_special_token_161|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128170" : {
"content" : "<|reserved_special_token_162|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128171" : {
"content" : "<|reserved_special_token_163|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128172" : {
"content" : "<|reserved_special_token_164|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128173" : {
"content" : "<|reserved_special_token_165|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128174" : {
"content" : "<|reserved_special_token_166|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128175" : {
"content" : "<|reserved_special_token_167|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128176" : {
"content" : "<|reserved_special_token_168|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128177" : {
"content" : "<|reserved_special_token_169|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128178" : {
"content" : "<|reserved_special_token_170|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128179" : {
"content" : "<|reserved_special_token_171|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128180" : {
"content" : "<|reserved_special_token_172|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128181" : {
"content" : "<|reserved_special_token_173|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128182" : {
"content" : "<|reserved_special_token_174|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128183" : {
"content" : "<|reserved_special_token_175|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128184" : {
"content" : "<|reserved_special_token_176|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128185" : {
"content" : "<|reserved_special_token_177|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128186" : {
"content" : "<|reserved_special_token_178|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128187" : {
"content" : "<|reserved_special_token_179|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128188" : {
"content" : "<|reserved_special_token_180|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128189" : {
"content" : "<|reserved_special_token_181|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128190" : {
"content" : "<|reserved_special_token_182|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128191" : {
"content" : "<|reserved_special_token_183|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128192" : {
"content" : "<|reserved_special_token_184|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128193" : {
"content" : "<|reserved_special_token_185|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128194" : {
"content" : "<|reserved_special_token_186|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128195" : {
"content" : "<|reserved_special_token_187|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128196" : {
"content" : "<|reserved_special_token_188|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128197" : {
"content" : "<|reserved_special_token_189|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128198" : {
"content" : "<|reserved_special_token_190|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128199" : {
"content" : "<|reserved_special_token_191|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128200" : {
"content" : "<|reserved_special_token_192|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128201" : {
"content" : "<|reserved_special_token_193|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128202" : {
"content" : "<|reserved_special_token_194|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128203" : {
"content" : "<|reserved_special_token_195|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128204" : {
"content" : "<|reserved_special_token_196|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128205" : {
"content" : "<|reserved_special_token_197|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128206" : {
"content" : "<|reserved_special_token_198|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128207" : {
"content" : "<|reserved_special_token_199|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128208" : {
"content" : "<|reserved_special_token_200|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128209" : {
"content" : "<|reserved_special_token_201|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128210" : {
"content" : "<|reserved_special_token_202|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128211" : {
"content" : "<|reserved_special_token_203|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128212" : {
"content" : "<|reserved_special_token_204|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128213" : {
"content" : "<|reserved_special_token_205|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128214" : {
"content" : "<|reserved_special_token_206|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128215" : {
"content" : "<|reserved_special_token_207|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128216" : {
"content" : "<|reserved_special_token_208|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128217" : {
"content" : "<|reserved_special_token_209|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128218" : {
"content" : "<|reserved_special_token_210|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128219" : {
"content" : "<|reserved_special_token_211|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128220" : {
"content" : "<|reserved_special_token_212|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128221" : {
"content" : "<|reserved_special_token_213|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128222" : {
"content" : "<|reserved_special_token_214|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128223" : {
"content" : "<|reserved_special_token_215|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128224" : {
"content" : "<|reserved_special_token_216|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128225" : {
"content" : "<|reserved_special_token_217|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128226" : {
"content" : "<|reserved_special_token_218|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128227" : {
"content" : "<|reserved_special_token_219|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128228" : {
"content" : "<|reserved_special_token_220|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128229" : {
"content" : "<|reserved_special_token_221|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128230" : {
"content" : "<|reserved_special_token_222|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128231" : {
"content" : "<|reserved_special_token_223|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128232" : {
"content" : "<|reserved_special_token_224|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128233" : {
"content" : "<|reserved_special_token_225|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128234" : {
"content" : "<|reserved_special_token_226|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128235" : {
"content" : "<|reserved_special_token_227|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128236" : {
"content" : "<|reserved_special_token_228|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128237" : {
"content" : "<|reserved_special_token_229|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128238" : {
"content" : "<|reserved_special_token_230|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128239" : {
"content" : "<|reserved_special_token_231|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128240" : {
"content" : "<|reserved_special_token_232|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128241" : {
"content" : "<|reserved_special_token_233|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128242" : {
"content" : "<|reserved_special_token_234|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128243" : {
"content" : "<|reserved_special_token_235|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128244" : {
"content" : "<|reserved_special_token_236|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128245" : {
"content" : "<|reserved_special_token_237|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128246" : {
"content" : "<|reserved_special_token_238|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128247" : {
"content" : "<|reserved_special_token_239|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128248" : {
"content" : "<|reserved_special_token_240|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128249" : {
"content" : "<|reserved_special_token_241|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128250" : {
"content" : "<|reserved_special_token_242|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128251" : {
"content" : "<|reserved_special_token_243|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128252" : {
"content" : "<|reserved_special_token_244|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128253" : {
"content" : "<|reserved_special_token_245|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128254" : {
"content" : "<|reserved_special_token_246|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
} ,
"128255" : {
"content" : "<|reserved_special_token_247|>" ,
"lstrip" : false ,
"normalized" : false ,
"rstrip" : false ,
"single_word" : false ,
"special" : true
}
} ,
"bos_token" : "<|begin_of_text|>" ,
2024-08-03 16:28:39 +00:00
"chat_template" : "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\"
2024-07-26 11:33:04 +00:00
"clean_up_tokenization_spaces" : true ,
"eos_token" : "<|eot_id|>" ,
"model_input_names" : [
"input_ids" ,
"attention_mask"
] ,
"model_max_length" : 131072 ,
"pad_token" : "<|eot_id|>" ,
"tokenizer_class" : "PreTrainedTokenizerFast"
}