Files
Bharat-Tiny-LLM-v2/pipeline_report.json

1814 lines
31 KiB
JSON
Raw Normal View History

{
"tokens_added": 300,
"token_ids": [
151665,
151666,
151667,
151668,
151669,
151670,
151671,
151672,
151673,
151674,
151675,
151676,
151677,
151678,
151679,
151680,
151681,
151682,
151683,
151684,
151685,
151686,
151687,
151688,
151689,
151690,
151691,
151692,
151693,
151694,
151695,
151696,
151697,
151698,
151699,
151700,
151701,
151702,
151703,
151704,
151705,
151706,
151707,
151708,
151709,
151710,
151711,
151712,
151713,
151714,
151715,
151716,
151717,
151718,
151719,
151720,
151721,
151722,
151723,
151724,
151725,
151726,
151727,
151728,
151729,
151730,
151731,
151732,
151733,
151734,
151735,
151736,
151737,
151738,
151739,
151740,
151741,
151742,
151743,
151744,
151745,
151746,
151747,
151748,
151749,
151750,
151751,
151752,
151753,
151754,
151755,
151756,
151757,
151758,
151759,
151760,
151761,
151762,
151763,
151764,
151765,
151766,
151767,
151768,
151769,
151770,
151771,
151772,
151773,
151774,
151775,
151776,
151777,
151778,
151779,
151780,
151781,
151782,
151783,
151784,
151785,
151786,
151787,
151788,
151789,
151790,
151791,
151792,
151793,
151794,
151795,
151796,
151797,
151798,
151799,
151800,
151801,
151802,
151803,
151804,
151805,
151806,
151807,
151808,
151809,
151810,
151811,
151812,
151813,
151814,
151815,
151816,
151817,
151818,
151819,
151820,
151821,
151822,
151823,
151824,
151825,
151826,
151827,
151828,
151829,
151830,
151831,
151832,
151833,
151834,
151835,
151836,
151837,
151838,
151839,
151840,
151841,
151842,
151843,
151844,
151845,
151846,
151847,
151848,
151849,
151850,
151851,
151852,
151853,
151854,
151855,
151856,
151857,
151858,
151859,
151860,
151861,
151862,
151863,
151864,
151865,
151866,
151867,
151868,
151869,
151870,
151871,
151872,
151873,
151874,
151875,
151876,
151877,
151878,
151879,
151880,
151881,
151882,
151883,
151884,
151885,
151886,
151887,
151888,
151889,
151890,
151891,
151892,
151893,
151894,
151895,
151896,
151897,
151898,
151899,
151900,
151901,
151902,
151903,
151904,
151905,
151906,
151907,
151908,
151909,
151910,
151911,
151912,
151913,
151914,
151915,
151916,
151917,
151918,
151919,
151920,
151921,
151922,
151923,
151924,
151925,
151926,
151927,
151928,
151929,
151930,
151931,
151932,
151933,
151934,
151935,
151936,
151937,
151938,
151939,
151940,
151941,
151942,
151943,
151944,
151945,
151946,
151947,
151948,
151949,
151950,
151951,
151952,
151953,
151954,
151955,
151956,
151957,
151958,
151959,
151960,
151961,
151962,
151963,
151964
],
"old_vocab_size": 151936,
"new_vocab_size": 152236,
"compression_chars_per_token_baseline": 434.975,
"compression_chars_per_token_improved": 657.316,
"compression_improvement_pct": 51.1,
"token_savings_pct": 33.8,
"init_method": "mean",
"selected_tokens": [
{
"token": "है",
"saved_tokens": 2,
"frequency": 33145
},
{
"token": "हैं",
"saved_tokens": 3,
"frequency": 11690
},
{
"token": "है।",
"saved_tokens": 3,
"frequency": 6916
},
{
"token": "ैं",
"saved_tokens": 2,
"frequency": 14416
},
{
"token": "हैं।",
"saved_tokens": 4,
"frequency": 3376
},
{
"token": "ज़रूरी",
"saved_tokens": 7,
"frequency": 971
},
{
"token": "में",
"saved_tokens": 2,
"frequency": 11848
},
{
"token": "ज़रूर",
"saved_tokens": 6,
"frequency": 1292
},
{
"token": "क्या",
"saved_tokens": 3,
"frequency": 5063
},
{
"token": "नहीं",
"saved_tokens": 3,
"frequency": 3928
},
{
"token": "ैसे",
"saved_tokens": 3,
"frequency": 3904
},
{
"token": "़रूरी",
"saved_tokens": 6,
"frequency": 971
},
{
"token": "ज़्याद",
"saved_tokens": 6,
"frequency": 900
},
{
"token": "़्यादा",
"saved_tokens": 6,
"frequency": 899
},
{
"token": "ज़रू",
"saved_tokens": 5,
"frequency": 1292
},
{
"token": "़रूर",
"saved_tokens": 5,
"frequency": 1292
},
{
"token": "कैसे",
"saved_tokens": 4,
"frequency": 2002
},
{
"token": "्या",
"saved_tokens": 2,
"frequency": 7829
},
{
"token": "पॉइंट",
"saved_tokens": 5,
"frequency": 1238
},
{
"token": "ैं।",
"saved_tokens": 3,
"frequency": 3376
},
{
"token": "ज़",
"saved_tokens": 2,
"frequency": 7267
},
{
"token": "ै।",
"saved_tokens": 2,
"frequency": 6916
},
{
"token": "फॉर्म",
"saved_tokens": 5,
"frequency": 1092
},
{
"token": "फॉर्मे",
"saved_tokens": 6,
"frequency": 733
},
{
"token": "जैसे",
"saved_tokens": 4,
"frequency": 1585
},
{
"token": "तुम्हा",
"saved_tokens": 5,
"frequency": 1011
},
{
"token": "ुम्हार",
"saved_tokens": 5,
"frequency": 1011
},
{
"token": "हूँ",
"saved_tokens": 4,
"frequency": 1578
},
{
"token": "क्य",
"saved_tokens": 2,
"frequency": 6078
},
{
"token": "स्ट",
"saved_tokens": 2,
"frequency": 6004
},
{
"token": "तुम्ह",
"saved_tokens": 4,
"frequency": 1494
},
{
"token": "बहुत",
"saved_tokens": 3,
"frequency": 2602
},
{
"token": "यूज़",
"saved_tokens": 5,
"frequency": 930
},
{
"token": "ज़्या",
"saved_tokens": 5,
"frequency": 901
},
{
"token": "़्याद",
"saved_tokens": 5,
"frequency": 900
},
{
"token": "ॉइंट्स",
"saved_tokens": 6,
"frequency": 598
},
{
"token": "या",
"saved_tokens": 1,
"frequency": 21373
},
{
"token": "पॉइंट्",
"saved_tokens": 6,
"frequency": 588
},
{
"token": "ॉर्म",
"saved_tokens": 4,
"frequency": 1300
},
{
"token": "़रू",
"saved_tokens": 4,
"frequency": 1292
},
{
"token": "कैस",
"saved_tokens": 3,
"frequency": 2253
},
{
"token": "ॉइंट",
"saved_tokens": 4,
"frequency": 1260
},
{
"token": "प्लान",
"saved_tokens": 4,
"frequency": 1254
},
{
"token": "ें",
"saved_tokens": 1,
"frequency": 20044
},
{
"token": "पॉइं",
"saved_tokens": 4,
"frequency": 1239
},
{
"token": "ॉर्मे",
"saved_tokens": 5,
"frequency": 782
},
{
"token": "ैस",
"saved_tokens": 2,
"frequency": 4850
},
{
"token": "मैं",
"saved_tokens": 3,
"frequency": 2154
},
{
"token": "्यादा",
"saved_tokens": 4,
"frequency": 1184
},
{
"token": "थोड़ा",
"saved_tokens": 5,
"frequency": 751
},
{
"token": "ूँ",
"saved_tokens": 3,
"frequency": 2061
},
{
"token": "ड़",
"saved_tokens": 2,
"frequency": 4580
},
{
"token": "जैस",
"saved_tokens": 3,
"frequency": 1969
},
{
"token": "ॉर्मेश",
"saved_tokens": 6,
"frequency": 489
},
{
"token": "फॉर्",
"saved_tokens": 4,
"frequency": 1093
},
{
"token": "हीं",
"saved_tokens": 2,
"frequency": 4264
},
{
"token": "करने",
"saved_tokens": 3,
"frequency": 1866
},
{
"token": "क्",
"saved_tokens": 1,
"frequency": 16562
},
{
"token": "रूरी",
"saved_tokens": 4,
"frequency": 1033
},
{
"token": "करना",
"saved_tokens": 3,
"frequency": 1822
},
{
"token": "से",
"saved_tokens": 1,
"frequency": 16360
},
{
"token": "होता",
"saved_tokens": 3,
"frequency": 1807
},
{
"token": "ुम्हा",
"saved_tokens": 4,
"frequency": 1011
},
{
"token": "म्हार",
"saved_tokens": 4,
"frequency": 1011
},
{
"token": "कैज़ुअ",
"saved_tokens": 7,
"frequency": 327
},
{
"token": "ैज़ुअल",
"saved_tokens": 7,
"frequency": 327
},
{
"token": "थोड़",
"saved_tokens": 4,
"frequency": 988
},
{
"token": "नही",
"saved_tokens": 2,
"frequency": 3929
},
{
"token": "करन",
"saved_tokens": 2,
"frequency": 3873
},
{
"token": "ूज़",
"saved_tokens": 4,
"frequency": 966
},
{
"token": "चाहिए",
"saved_tokens": 4,
"frequency": 942
},
{
"token": "करते",
"saved_tokens": 3,
"frequency": 1673
},
{
"token": "बिल्कु",
"saved_tokens": 5,
"frequency": 602
},
{
"token": "िल्कुल",
"saved_tokens": 5,
"frequency": 602
},
{
"token": "िया",
"saved_tokens": 2,
"frequency": 3750
},
{
"token": "ॉइंट्",
"saved_tokens": 5,
"frequency": 599
},
{
"token": "कर",
"saved_tokens": 1,
"frequency": 14963
},
{
"token": "लिए",
"saved_tokens": 2,
"frequency": 3722
},
{
"token": "इंडिय",
"saved_tokens": 4,
"frequency": 929
},
{
"token": "मे",
"saved_tokens": 1,
"frequency": 14729
},
{
"token": "ज़्य",
"saved_tokens": 4,
"frequency": 920
},
{
"token": "होत",
"saved_tokens": 2,
"frequency": 3646
},
{
"token": "वपूर्ण",
"saved_tokens": 6,
"frequency": 405
},
{
"token": "हत्वपू",
"saved_tokens": 6,
"frequency": 403
},
{
"token": "त्वपूर",
"saved_tokens": 6,
"frequency": 402
},
{
"token": "्वपूर्",
"saved_tokens": 6,
"frequency": 402
},
{
"token": "़्या",
"saved_tokens": 4,
"frequency": 904
},
{
"token": "्यू",
"saved_tokens": 3,
"frequency": 1600
},
{
"token": "बढ़िया",
"saved_tokens": 6,
"frequency": 399
},
{
"token": "हाँ",
"saved_tokens": 2,
"frequency": 3582
},
{
"token": "के",
"saved_tokens": 1,
"frequency": 14303
},
{
"token": "ॉर्",
"saved_tokens": 3,
"frequency": 1573
},
{
"token": "का",
"saved_tokens": 1,
"frequency": 13939
},
{
"token": "कॉन्टे",
"saved_tokens": 6,
"frequency": 385
},
{
"token": "ड़ा",
"saved_tokens": 3,
"frequency": 1531
},
{
"token": "म्हारी",
"saved_tokens": 5,
"frequency": 549
},
{
"token": "वैल्यू",
"saved_tokens": 7,
"frequency": 280
},
{
"token": "ॉन्टेक",
"saved_tokens": 6,
"frequency": 381
},
{
"token": "ार",
"saved_tokens": 1,
"frequency": 13648
},
{
"token": "्य",
"saved_tokens": 1,
"frequency": 13535
},
{
"token": "ज़र",
"saved_tokens": 3,
"frequency": 1499
},
{
"token": "तुम्",
"saved_tokens": 3,
"frequency": 1495
},
{
"token": "इंडिया",
"saved_tokens": 5,
"frequency": 538
},
{
"token": "ुम्ह",
"saved_tokens": 3,
"frequency": 1494
},
{
"token": "प्ला",
"saved_tokens": 3,
"frequency": 1485
},
{
"token": "यार",
"saved_tokens": 2,
"frequency": 3333
},
{
"token": "रूर",
"saved_tokens": 3,
"frequency": 1470
},
{
"token": "र्",
"saved_tokens": 1,
"frequency": 13216
},
{
"token": "ूर",
"saved_tokens": 2,
"frequency": 3293
},
{
"token": "चेकलिस",
"saved_tokens": 5,
"frequency": 526
},
{
"token": "ेकलिस्",
"saved_tokens": 5,
"frequency": 526
},
{
"token": "कलिस्ट",
"saved_tokens": 5,
"frequency": 526
},
{
"token": "स्ट्",
"saved_tokens": 3,
"frequency": 1455
},
{
"token": "र्मेशन",
"saved_tokens": 5,
"frequency": 521
},
{
"token": "ना",
"saved_tokens": 1,
"frequency": 13016
},
{
"token": "सबसे",
"saved_tokens": 3,
"frequency": 1445
},
{
"token": "ूरी",
"saved_tokens": 3,
"frequency": 1442
},
{
"token": "इन्फॉर",
"saved_tokens": 6,
"frequency": 359
},
{
"token": "न्फॉर्",
"saved_tokens": 6,
"frequency": 359
},
{
"token": "्फॉर्म",
"saved_tokens": 6,
"frequency": 359
},
{
"token": "कै",
"saved_tokens": 2,
"frequency": 3205
},
{
"token": "एकदम",
"saved_tokens": 3,
"frequency": 1413
},
{
"token": "अच्छ",
"saved_tokens": 3,
"frequency": 1406
},
{
"token": "पूर",
"saved_tokens": 3,
"frequency": 1386
},
{
"token": "रू",
"saved_tokens": 2,
"frequency": 3106
},
{
"token": "सिर्फ",
"saved_tokens": 4,
"frequency": 775
},
{
"token": "क्यों",
"saved_tokens": 4,
"frequency": 771
},
{
"token": "स्ट्र",
"saved_tokens": 3,
"frequency": 1370
},
{
"token": "करत",
"saved_tokens": 2,
"frequency": 3072
},
{
"token": "ोड़ा",
"saved_tokens": 4,
"frequency": 767
},
{
"token": "ता",
"saved_tokens": 1,
"frequency": 12200
},
{
"token": "तुम्हे",
"saved_tokens": 5,
"frequency": 482
},
{
"token": "ुम्हें",
"saved_tokens": 5,
"frequency": 481
},
{
"token": "अच्छा",
"saved_tokens": 4,
"frequency": 751
},
{
"token": "बढ़",
"saved_tokens": 3,
"frequency": 1329
},
{
"token": "स्",
"saved_tokens": 1,
"frequency": 11934
},
{
"token": "िस्ट",
"saved_tokens": 3,
"frequency": 1314
},
{
"token": "ीज़",
"saved_tokens": 3,
"frequency": 1310
},
{
"token": "कैज़ु",
"saved_tokens": 6,
"frequency": 327
},
{
"token": "ैज़ुअ",
"saved_tokens": 6,
"frequency": 327
},
{
"token": "चीज़",
"saved_tokens": 4,
"frequency": 735
},
{
"token": "तैयार",
"saved_tokens": 5,
"frequency": 468
},
{
"token": "सकते",
"saved_tokens": 3,
"frequency": 1298
},
{
"token": "्ट",
"saved_tokens": 1,
"frequency": 11668
},
{
"token": "मै",
"saved_tokens": 2,
"frequency": 2916
},
{
"token": "ान",
"saved_tokens": 1,
"frequency": 11540
},
{
"token": "करें",
"saved_tokens": 3,
"frequency": 1282
},
{
"token": "हो",
"saved_tokens": 1,
"frequency": 11520
},
{
"token": "यूज",
"saved_tokens": 3,
"frequency": 1276
},
{
"token": "ने",
"saved_tokens": 1,
"frequency": 11388
},
{
"token": "ॉइं",
"saved_tokens": 3,
"frequency": 1264
},
{
"token": "्लान",
"saved_tokens": 3,
"frequency": 1261
},
{
"token": "करूँगा",
"saved_tokens": 7,
"frequency": 231
},
{
"token": "कॉन्ट",
"saved_tokens": 5,
"frequency": 452
},
{
"token": "ोड़",
"saved_tokens": 3,
"frequency": 1242
},
{
"token": "पॉइ",
"saved_tokens": 3,
"frequency": 1240
},
{
"token": "़ा",
"saved_tokens": 2,
"frequency": 2768
},
{
"token": "िंग",
"saved_tokens": 2,
"frequency": 2766
},
{
"token": "्स",
"saved_tokens": 1,
"frequency": 11009
},
{
"token": "यू",
"saved_tokens": 2,
"frequency": 2750
},
{
"token": "पूर्ण",
"saved_tokens": 5,
"frequency": 440
},
{
"token": "लिस्ट",
"saved_tokens": 4,
"frequency": 687
},
{
"token": "क्स",
"saved_tokens": 2,
"frequency": 2724
},
{
"token": "जानकार",
"saved_tokens": 5,
"frequency": 428
},
{
"token": "्याद",
"saved_tokens": 3,
"frequency": 1188
},
{
"token": "बहु",
"saved_tokens": 2,
"frequency": 2666
},
{
"token": "यादा",
"saved_tokens": 3,
"frequency": 1184
},
{
"token": "और",
"saved_tokens": 1,
"frequency": 10621
},
{
"token": "ानकारी",
"saved_tokens": 5,
"frequency": 424
},
{
"token": "फॉर",
"saved_tokens": 3,
"frequency": 1171
},
{
"token": "हिंदी",
"saved_tokens": 4,
"frequency": 658
},
{
"token": "हुत",
"saved_tokens": 2,
"frequency": 2603
},
{
"token": "ऑनलाइन",
"saved_tokens": 5,
"frequency": 416
},
{
"token": "कुछ",
"saved_tokens": 2,
"frequency": 2599
},
{
"token": "ेक्स्ट",
"saved_tokens": 5,
"frequency": 415
},
{
"token": "को",
"saved_tokens": 1,
"frequency": 10340
},
{
"token": "तरीके",
"saved_tokens": 4,
"frequency": 637
},
{
"token": "वाल",
"saved_tokens": 2,
"frequency": 2534
},
{
"token": "वपूर्",
"saved_tokens": 5,
"frequency": 405
},
{
"token": "चीज़ें",
"saved_tokens": 6,
"frequency": 281
},
{
"token": "ैल्यू",
"saved_tokens": 6,
"frequency": 280
},
{
"token": "त्वपू",
"saved_tokens": 5,
"frequency": 403
},
{
"token": "महत्वप",
"saved_tokens": 5,
"frequency": 403
},
{
"token": "तरीक",
"saved_tokens": 3,
"frequency": 1117
},
{
"token": "्वपूर",
"saved_tokens": 5,
"frequency": 402
},
{
"token": "टेक्स्",
"saved_tokens": 5,
"frequency": 402
},
{
"token": "ढ़िया",
"saved_tokens": 5,
"frequency": 400
},
{
"token": "बढ़िय",
"saved_tokens": 5,
"frequency": 399
},
{
"token": "इंट्स",
"saved_tokens": 4,
"frequency": 623
},
{
"token": "पोर्ट",
"saved_tokens": 4,
"frequency": 620
},
{
"token": "ते",
"saved_tokens": 1,
"frequency": 9876
},
{
"token": "्टेक्स",
"saved_tokens": 5,
"frequency": 391
},
{
"token": "बड़",
"saved_tokens": 3,
"frequency": 1085
},
{
"token": "सकत",
"saved_tokens": 2,
"frequency": 2441
},
{
"token": "ॉन्टे",
"saved_tokens": 5,
"frequency": 387
},
{
"token": "बिल्क",
"saved_tokens": 4,
"frequency": 602
},
{
"token": "िल्कु",
"saved_tokens": 4,
"frequency": 602
},
{
"token": "ल्कुल",
"saved_tokens": 4,
"frequency": 602
},
{
"token": "कानून",
"saved_tokens": 5,
"frequency": 385
},
{
"token": "उन्हें",
"saved_tokens": 5,
"frequency": 385
},
{
"token": "इंडियन",
"saved_tokens": 5,
"frequency": 384
},
{
"token": "ध्यान",
"saved_tokens": 4,
"frequency": 597
},
{
"token": "न्टेक्",
"saved_tokens": 5,
"frequency": 381
},
{
"token": "मेंट",
"saved_tokens": 3,
"frequency": 1051
},
{
"token": "देख",
"saved_tokens": 2,
"frequency": 2352
},
{
"token": "ट्स",
"saved_tokens": 2,
"frequency": 2345
},
{
"token": "स्ट्रक",
"saved_tokens": 4,
"frequency": 585
},
{
"token": "्ट्रक्",
"saved_tokens": 4,
"frequency": 585
},
{
"token": "ूँगा",
"saved_tokens": 5,
"frequency": 370
},
{
"token": "कॉन्",
"saved_tokens": 4,
"frequency": 576
},
{
"token": "्हार",
"saved_tokens": 3,
"frequency": 1012
},
{
"token": "ंट्स",
"saved_tokens": 3,
"frequency": 1011
},
{
"token": "म्हा",
"saved_tokens": 3,
"frequency": 1011
},
{
"token": "प्ल",
"saved_tokens": 2,
"frequency": 2262
},
{
"token": "्ट्",
"saved_tokens": 2,
"frequency": 2260
},
{
"token": "ॉर्मेट",
"saved_tokens": 6,
"frequency": 250
},
{
"token": "देखो",
"saved_tokens": 3,
"frequency": 999
},
{
"token": "्हें",
"saved_tokens": 3,
"frequency": 998
},
{
"token": "इन्फॉ",
"saved_tokens": 5,
"frequency": 359
},
{
"token": "न्फॉर",
"saved_tokens": 5,
"frequency": 359
},
{
"token": "्फॉर्",
"saved_tokens": 5,
"frequency": 359
},
{
"token": "होती",
"saved_tokens": 3,
"frequency": 996
},
{
"token": "ज़्",
"saved_tokens": 3,
"frequency": 989
},
{
"token": "करेंगे",
"saved_tokens": 5,
"frequency": 353
},
{
"token": "तुम",
"saved_tokens": 2,
"frequency": 2200
},
{
"token": "्हारी",
"saved_tokens": 4,
"frequency": 550
},
{
"token": "कार",
"saved_tokens": 2,
"frequency": 2184
},
{
"token": "ज़ुअल",
"saved_tokens": 5,
"frequency": 349
},
{
"token": "पड़ता",
"saved_tokens": 5,
"frequency": 347
},
{
"token": "ंडिया",
"saved_tokens": 4,
"frequency": 541
},
{
"token": "प्",
"saved_tokens": 1,
"frequency": 8617
},
{
"token": "इंडि",
"saved_tokens": 3,
"frequency": 957
},
{
"token": "कि",
"saved_tokens": 1,
"frequency": 8611
},
{
"token": "्यों",
"saved_tokens": 3,
"frequency": 951
},
{
"token": "करता",
"saved_tokens": 3,
"frequency": 951
},
{
"token": "रने",
"saved_tokens": 2,
"frequency": 2139
},
{
"token": "ाना",
"saved_tokens": 2,
"frequency": 2134
},
{
"token": "चाहि",
"saved_tokens": 3,
"frequency": 943
},
{
"token": "ाहिए",
"saved_tokens": 3,
"frequency": 942
},
{
"token": "रूँगा",
"saved_tokens": 6,
"frequency": 235
},
{
"token": "ढ़",
"saved_tokens": 2,
"frequency": 2112
},
{
"token": "पूर्",
"saved_tokens": 4,
"frequency": 528
},
{
"token": "ंडिय",
"saved_tokens": 3,
"frequency": 937
},
{
"token": "ेक्ट",
"saved_tokens": 3,
"frequency": 937
},
{
"token": "ारी",
"saved_tokens": 2,
"frequency": 2104
},
{
"token": "चेकलि",
"saved_tokens": 4,
"frequency": 526
},
{
"token": "ेकलिस",
"saved_tokens": 4,
"frequency": 526
},
{
"token": "कलिस्",
"saved_tokens": 4,
"frequency": 526
},
{
"token": "जै",
"saved_tokens": 2,
"frequency": 2101
},
{
"token": "बना",
"saved_tokens": 2,
"frequency": 2097
},
{
"token": "डिया",
"saved_tokens": 3,
"frequency": 932
},
{
"token": "बड़ा",
"saved_tokens": 4,
"frequency": 523
},
{
"token": "ेक्",
"saved_tokens": 2,
"frequency": 2088
},
{
"token": "र्मेश",
"saved_tokens": 4,
"frequency": 522
},
{
"token": "करूँग",
"saved_tokens": 6,
"frequency": 232
},
{
"token": "्मेशन",
"saved_tokens": 4,
"frequency": 521
},
{
"token": "ज़रूरत",
"saved_tokens": 7,
"frequency": 170
},
{
"token": "़्य",
"saved_tokens": 3,
"frequency": 924
},
{
"token": "न्हें",
"saved_tokens": 4,
"frequency": 517
},
{
"token": "ही",
"saved_tokens": 1,
"frequency": 8269
},
{
"token": "इंस्ट्",
"saved_tokens": 5,
"frequency": 330
},
{
"token": "ेस्ट",
"saved_tokens": 3,
"frequency": 915
},
{
"token": "रे",
"saved_tokens": 1,
"frequency": 8188
},
{
"token": "हा",
"saved_tokens": 1,
"frequency": 8186
},
{
"token": "होगा",
"saved_tokens": 3,
"frequency": 909
},
{
"token": "कैज़",
"saved_tokens": 5,
"frequency": 327
},
{
"token": "ैज़ु",
"saved_tokens": 5,
"frequency": 327
},
{
"token": "बात",
"saved_tokens": 2,
"frequency": 2029
},
{
"token": "ेंट",
"saved_tokens": 2,
"frequency": 2028
},
{
"token": "मिल",
"saved_tokens": 2,
"frequency": 2026
},
{
"token": "ों",
"saved_tokens": 1,
"frequency": 8055
},
{
"token": "अपन",
"saved_tokens": 2,
"frequency": 2010
},
{
"token": "पहले",
"saved_tokens": 3,
"frequency": 893
},
{
"token": "टाइम",
"saved_tokens": 3,
"frequency": 891
},
{
"token": "र्म",
"saved_tokens": 2,
"frequency": 1998
},
{
"token": "ैक्ट",
"saved_tokens": 4,
"frequency": 499
},
{
"token": "ेंगे",
"saved_tokens": 3,
"frequency": 886
},
{
"token": "री",
"saved_tokens": 1,
"frequency": 7923
},
{
"token": "क्यो",
"saved_tokens": 3,
"frequency": 879
},
{
"token": "िस्",
"saved_tokens": 2,
"frequency": 1965
},
{
"token": "अपनी",
"saved_tokens": 3,
"frequency": 871
},
{
"token": "लोगों",
"saved_tokens": 4,
"frequency": 489
},
{
"token": "जाता",
"saved_tokens": 3,
"frequency": 866
},
{
"token": "रना",
"saved_tokens": 2,
"frequency": 1942
},
{
"token": "पड़",
"saved_tokens": 3,
"frequency": 863
},
{
"token": "आपक",
"saved_tokens": 2,
"frequency": 1941
},
{
"token": "ुम्हे",
"saved_tokens": 4,
"frequency": 482
},
{
"token": "क्स्ट",
"saved_tokens": 4,
"frequency": 482
},
{
"token": "म्हें",
"saved_tokens": 4,
"frequency": 481
},
{
"token": "ैक्",
"saved_tokens": 3,
"frequency": 855
},
{
"token": "पॉ",
"saved_tokens": 2,
"frequency": 1915
},
{
"token": "ार्",
"saved_tokens": 2,
"frequency": 1907
},
{
"token": "रें",
"saved_tokens": 2,
"frequency": 1901
},
{
"token": "इस्तेम",
"saved_tokens": 5,
"frequency": 304
},
{
"token": "स्तेमा",
"saved_tokens": 5,
"frequency": 304
},
{
"token": "्तेमाल",
"saved_tokens": 5,
"frequency": 303
},
{
"token": "अक्सर",
"saved_tokens": 4,
"frequency": 473
},
{
"token": "होते",
"saved_tokens": 3,
"frequency": 840
},
{
"token": "ेशन",
"saved_tokens": 2,
"frequency": 1888
},
{
"token": "करूँ",
"saved_tokens": 5,
"frequency": 301
}
]
}