36 lines
1.1 KiB
JSON
36 lines
1.1 KiB
JSON
{
|
|
"vocab_size": 64000,
|
|
"total_texts": 6179192,
|
|
"corpus_size_bytes": 17442976221,
|
|
"training_time_seconds": 1414.3693861961365,
|
|
"data_sources": {
|
|
"01_논문_학술": 500042,
|
|
"02_도서_말뭉치": 500003,
|
|
"03_전문분야": 509076,
|
|
"04_웹_뉴스": 501738,
|
|
"05_법률_행정": 500000,
|
|
"06_의료_건강": 500000,
|
|
"07_교육_지식": 214238,
|
|
"culturax_korean": 200000,
|
|
"ko_alpaca_instructions": 21149,
|
|
"korean_textbooks_claude_evol": 200000,
|
|
"korean_textbooks_code_alpaca": 64006,
|
|
"korean_textbooks_instructions": 200000,
|
|
"korean_textbooks_mmlu_all": 97697,
|
|
"korean_textbooks_tiny": 200000,
|
|
"korean_textbooks_wikidata": 127470,
|
|
"wikipedia_korean_2024": 200000,
|
|
"ko_alpaca_v1.1a": 21149,
|
|
"ko_orca": 34163,
|
|
"korean_webtext": 200000,
|
|
"mc4_korean": 200000,
|
|
"namuwiki": 200000,
|
|
"open_ko_instructions": 200000,
|
|
"cosmopedia_stories_science": 200000,
|
|
"magpie_reasoning_150k": 148703,
|
|
"math_instruct": 200000,
|
|
"python_code_instructions_18k": 18609
|
|
},
|
|
"date": "2026-08-29 19:30:06",
|
|
"repo_id": "cooler8/yejin-korean-tokenizer-64k"
|
|
} |