Files
cym_latn_10mb/added_tokens.json

1 line
25 KiB
JSON
Raw Permalink Normal View History

{"[XXXXX3]": 50007, "[XXXXX564]": 50568, "[XXXXX929]": 50933, "[XXXXX164]": 50168, "[XXXXX459]": 50463, "[XXXXX90]": 50094, "[XXXXX154]": 50158, "[XXXXX835]": 50839, "[XXXXX66]": 50070, "[XXXXX1031]": 51035, "[XXXXX1072]": 51076, "[XXXXX54]": 50058, "[XXXXX115]": 50119, "[XXXXX1174]": 51178, "[XXXXX397]": 50401, "[XXXXX373]": 50377, "[XXXXX100]": 50104, "[XXXXX644]": 50648, "[XXXXX1065]": 51069, "[XXXXX1177]": 51181, "[XXXXX35]": 50039, "[XXXXX944]": 50948, "[XXXXX616]": 50620, "[XXXXX838]": 50842, "[XXXXX782]": 50786, "[XXXXX1085]": 51089, "[XXXXX666]": 50670, "[XXXXX675]": 50679, "[XXXXX908]": 50912, "[XXXXX420]": 50424, "[XXXXX1036]": 51040, "[XXXXX716]": 50720, "[XXXXX166]": 50170, "[XXXXX424]": 50428, "[XXXXX740]": 50744, "[XXXXX553]": 50557, "[XXXXX561]": 50565, "[XXXXX590]": 50594, "[XXXXX429]": 50433, "[XXXXX516]": 50520, "[XXXXX546]": 50550, "[XXXXX611]": 50615, "[XXXXX744]": 50748, "[XXXXX531]": 50535, "[XXXXX718]": 50722, "[XXXXX763]": 50767, "[XXXXX664]": 50668, "[XXXXX426]": 50430, "[XXXXX389]": 50393, "[XXXXX1110]": 51114, "[XXXXX694]": 50698, "[XXXXX757]": 50761, "[XXXXX963]": 50967, "[XXXXX690]": 50694, "[XXXXX365]": 50369, "[XXXXX329]": 50333, "[XXXXX817]": 50821, "[XXXXX1189]": 51193, "[XXXXX388]": 50392, "[XXXXX1081]": 51085, "[XXXXX606]": 50610, "[XXXXX158]": 50162, "[XXXXX896]": 50900, "[XXXXX15]": 50019, "[XXXXX309]": 50313, "[XXXXX809]": 50813, "[XXXXX340]": 50344, "[XXXXX853]": 50857, "[XXXXX1117]": 51121, "[XXXXX1160]": 51164, "[XXXXX587]": 50591, "[XXXXX286]": 50290, "[XXXXX99]": 50103, "[XXXXX500]": 50504, "[XXXXX157]": 50161, "[XXXXX268]": 50272, "[XXXXX224]": 50228, "<pad>": 50002, "[XXXXX399]": 50403, "[XXXXX673]": 50677, "[XXXXX280]": 50284, "[XXXXX202]": 50206, "[XXXXX642]": 50646, "[XXXXX381]": 50385, "[XXXXX222]": 50226, "[XXXXX1037]": 51041, "[XXXXX406]": 50410, "[XXXXX1145]": 51149, "[XXXXX320]": 50324, "[XXXXX510]": 50514, "[XXXXX804]": 50808, "[XXXXX882]": 50886, "[XXXXX449]": 50453, "[XXXXX387]": 50391, "[XXXXX786]": 50790, "[XXXXX1178]": 51182, "[XXXXX490]": 50494, "[XXXXX1192]": 51196, "[XXXXX1028]": 51032, "[XXXXX1083]": 51087, "[XXXXX872]": 50876, "[XXXXX950]": 50954, "[XXXXX647]": 50651, "[XXXXX689]": 50693, "[XXXXX109]": 50113, "[XXXXX385]": 50389, "[XXXXX363]": 50367, "[XXXXX477]": 50481, "[XXXXX119]": 50123, "[XXXXX556]": 50560, "[XXXXX450]": 50454, "[XXXXX995]": 50999, "[XXXXX251]": 50255, "[XXXXX194]": 50198, "[XXXXX297]": 50301, "[XXXXX683]": 50687, "[XXXXX1130]": 51134, "[XXXXX76]": 50080, "[XXXXX881]": 50885, "[XXXXX728]": 50732, "[XXXXX455]": 50459, "[XXXXX1087]": 51091, "[XXXXX327]": 50331, "[XXXXX837]": 50841, "[XXXXX992]": 50996, "[XXXXX613]": 50617, "[XXXXX551]": 50555, "[XXXXX931]": 50935, "[XXXXX747]": 50751, "[XXXXX495]": 50499, "[XXXXX767]": 50771, "[XXXXX197]": 50201, "[XXXXX1014]": 51018, "[XXXXX21]": 50025, "[XXXXX581]": 50585, "[XXXXX902]": 50906, "[XXXXX38]": 50042, "[XXXXX981]": 50985, "[XXXXX660]": 50664, "[XXXXX1113]": 51117, "[XXXXX903]": 50907, "[XXXXX877]": 50881, "[XXXXX14]": 50018, "[XXXXX640]": 50644, "[XXXXX405]": 50409, "[XXXXX120]": 50124, "[XXXXX701]": 50705, "[XXXXX1158]": 51162, "[XXXXX511]": 50515, "[XXXXX247]": 50251, "[XXXXX998]": 51002, "[XXXXX463]": 50467, "[XXXXX196]": 50200, "[XXXXX1067]": 51071, "[XXXXX358]": 50362, "[XXXXX755]": 50759, "[XXXXX239]": 50243, "[XXXXX185]": 50189, "[XXXXX935]": 50939, "[XXXXX527]": 50531, "[XXXXX1195]": 51199, "[XXXXX1159]": 51163, "[XXXXX117]": 50121, "[XXXXX1103]": 51107, "[XXXXX517]": 50521, "[XXXXX870]": 50874, "[XXXXX9]": 50013, "[XXXXX306]": 50310, "[XXXXX1132]": 51136, "[XXXXX1017]": 51021, "[XXXXX656]": 50660, "[XXXXX1107]": 51111, "[XXXXX709]": 50713, "[XXXXX801]": 50805, "[XXXXX628]": 50632, "[XXXXX890]": 50894, "[XXXXX643]": 50647, "[XXXXX206]": 50210, "[XXXXX685]": 50689, "[XXXXX396]": 50400, "[XXXXX806]": 50810, "[XXXXX1040]": 51044, "[XXXXX1006]": 51010, "[XXXXX285]": 50289, "[XXXXX25]": 50029, "[XXXXX758]": 50762, "[XXXXX914]": 50918, "[XXXXX441]": 50445, "[XXXXX707]": 50711, "[XXXXX313]": 50317, "[XXXXX770]": 50774, "[XXXXX545]": 50549, "[XXXXX1064]": 51068, "[XXXXX753]": 50757, "[XXXXX