Files
zho_hans_1000mb/added_tokens.json

1 line
25 KiB
JSON
Raw Normal View History

{"<pad>": 50002, "[XXXXX642]": 50646, "[XXXXX625]": 50629, "[XXXXX467]": 50471, "[XXXXX1006]": 51010, "[XXXXX399]": 50403, "[XXXXX36]": 50040, "[XXXXX1005]": 51009, "[XXXXX650]": 50654, "[XXXXX206]": 50210, "[XXXXX853]": 50857, "[XXXXX347]": 50351, "[XXXXX415]": 50419, "[XXXXX602]": 50606, "[XXXXX1084]": 51088, "[XXXXX1089]": 51093, "[XXXXX939]": 50943, "[XXXXX226]": 50230, "[XXXXX200]": 50204, "[XXXXX509]": 50513, "[XXXXX5]": 50009, "[XXXXX65]": 50069, "[XXXXX158]": 50162, "[XXXXX786]": 50790, "[XXXXX362]": 50366, "[XXXXX865]": 50869, "[XXXXX636]": 50640, "[XXXXX730]": 50734, "[XXXXX23]": 50027, "[XXXXX889]": 50893, "[XXXXX281]": 50285, "[XXXXX201]": 50205, "[XXXXX1008]": 51012, "[XXXXX1076]": 51080, "[XXXXX374]": 50378, "[XXXXX271]": 50275, "[XXXXX535]": 50539, "[XXXXX351]": 50355, "[XXXXX58]": 50062, "[XXXXX869]": 50873, "[XXXXX227]": 50231, "[XXXXX122]": 50126, "[XXXXX389]": 50393, "[XXXXX91]": 50095, "[XXXXX1109]": 51113, "[XXXXX485]": 50489, "[XXXXX735]": 50739, "[XXXXX1027]": 51031, "[XXXXX278]": 50282, "[XXXXX119]": 50123, "[XXXXX425]": 50429, "[XXXXX909]": 50913, "[XXXXX567]": 50571, "[XXXXX1003]": 51007, "[XXXXX439]": 50443, "[XXXXX1087]": 51091, "[XXXXX1023]": 51027, "[XXXXX59]": 50063, "[XXXXX779]": 50783, "[XXXXX717]": 50721, "[XXXXX285]": 50289, "[XXXXX475]": 50479, "[XXXXX529]": 50533, "[XXXXX35]": 50039, "[XXXXX450]": 50454, "[XXXXX446]": 50450, "[XXXXX361]": 50365, "[XXXXX627]": 50631, "[XXXXX124]": 50128, "[XXXXX371]": 50375, "[XXXXX707]": 50711, "[XXXXX292]": 50296, "[XXXXX270]": 50274, "[XXXXX398]": 50402, "[XXXXX991]": 50995, "[XXXXX902]": 50906, "[XXXXX748]": 50752, "[XXXXX1130]": 51134, "[XXXXX640]": 50644, "[XXXXX839]": 50843, "[XXXXX578]": 50582, "[XXXXX750]": 50754, "[XXXXX925]": 50929, "[XXXXX785]": 50789, "[XXXXX639]": 50643, "[XXXXX41]": 50045, "[XXXXX1190]": 51194, "[XXXXX825]": 50829, "[XXXXX855]": 50859, "[XXXXX1099]": 51103, "[XXXXX1148]": 51152, "[XXXXX386]": 50390, "[XXXXX32]": 50036, "[XXXXX1055]": 51059, "[XXXXX511]": 50515, "[XXXXX60]": 50064, "[XXXXX298]": 50302, "[XXXXX708]": 50712, "[XXXXX1095]": 51099, "[XXXXX576]": 50580, "[XXXXX34]": 50038, "[XXXXX315]": 50319, "[XXXXX228]": 50232, "[XXXXX29]": 50033, "[XXXXX530]": 50534, "[XXXXX332]": 50336, "[XXXXX71]": 50075, "[XXXXX117]": 50121, "[XXXXX696]": 50700, "[XXXXX345]": 50349, "[XXXXX595]": 50599, "[XXXXX460]": 50464, "[XXXXX915]": 50919, "[XXXXX1044]": 51048, "[XXXXX1128]": 51132, "[XXXXX1184]": 51188, "[XXXXX131]": 50135, "[XXXXX811]": 50815, "[XXXXX861]": 50865, "[XXXXX102]": 50106, "[XXXXX1141]": 51145, "[XXXXX523]": 50527, "[XXXXX1000]": 51004, "[XXXXX1140]": 51144, "[XXXXX669]": 50673, "[XXXXX336]": 50340, "[XXXXX759]": 50763, "[XXXXX850]": 50854, "[XXXXX482]": 50486, "[XXXXX950]": 50954, "[XXXXX860]": 50864, "[XXXXX1082]": 51086, "[XXXXX551]": 50555, "[XXXXX757]": 50761, "[XXXXX985]": 50989, "[XXXXX331]": 50335, "[XXXXX500]": 50504, "[XXXXX990]": 50994, "[XXXXX1143]": 51147, "[XXXXX484]": 50488, "[XXXXX506]": 50510, "[XXXXX156]": 50160, "[XXXXX1127]": 51131, "[XXXXX242]": 50246, "[XXXXX930]": 50934, "[XXXXX981]": 50985, "[XXXXX936]": 50940, "[XXXXX868]": 50872, "[XXXXX230]": 50234, "[XXXXX291]": 50295, "[XXXXX822]": 50826, "[XXXXX899]": 50903, "[XXXXX187]": 50191, "[XXXXX713]": 50717, "[XXXXX176]": 50180, "[XXXXX996]": 51000, "[XXXXX462]": 50466, "[XXXXX359]": 50363, "[XXXXX687]": 50691, "[XXXXX773]": 50777, "[XXXXX126]": 50130, "[XXXXX704]": 50708, "[XXXXX241]": 50245, "[XXXXX115]": 50119, "[XXXXX83]": 50087, "[XXXXX245]": 50249, "[XXXXX1041]": 51045, "[XXXXX987]": 50991, "[XXXXX1167]": 51171, "[XXXXX764]": 50768, "[XXXXX26]": 50030, "[XXXXX286]": 50290, "[XXXXX590]": 50594, "[XXXXX182]": 50186, "[XXXXX451]": 50455, "[XXXXX512]": 50516, "[XXXXX100]": 50104, "[XXXXX327]": 50331, "[XXXXX784]": 50788, "[XXXXX38]": 50042, "[XXXXX195]": 50199, "[XXXXX1078]": 51082, "[XXXXX323]": 50327, "[XXXXX920]": 50924, "[XXXXX817]": 50821, "[XXXXX520]": 50524, "[XXXXX988]": 50992, "[XXXXX892]": 50896, "[XXXXX481]": 50485, "[XXXXX948]": 50952, "[XXXXX373]": 50377, "[XXXXX830]": 50834, "[XXXXX220]": 50224, "[XXXXX107]": 50111, "[XXXXX728]":