From b416fc0c1ee08bdd0d759bd69113fe9dc7003bff Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 18:04:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: eulogik/Bharat-Tiny-LLM-v2 Source: Original Platform --- .gitattributes | 38 + README.md | 139 ++ bharat_tiny_llm_v2_pipeline_diagram.jpg | 3 + bharat_tiny_llm_v2_tech_infographic.jpg | 3 + brahmi_continued_pretrain.ipynb | 394 +++++ brahmi_lora_qlora.ipynb | 342 +++++ chat_template.jinja | 54 + config.json | 62 + generation_config.json | 7 + model.safetensors | 3 + pipeline_report.json | 1814 +++++++++++++++++++++++ tokenizer.json | 3 + tokenizer_config.json | 30 + train_gold_v3.jsonl.gz | 3 + 14 files changed, 2895 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 bharat_tiny_llm_v2_pipeline_diagram.jpg create mode 100644 bharat_tiny_llm_v2_tech_infographic.jpg create mode 100644 brahmi_continued_pretrain.ipynb create mode 100644 brahmi_lora_qlora.ipynb create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 pipeline_report.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_gold_v3.jsonl.gz diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..58d8811 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,38 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +bharat_tiny_llm_v2_tech_infographic.jpg filter=lfs diff=lfs merge=lfs -text +bharat_tiny_llm_v2_pipeline_diagram.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..21e6bc7 --- /dev/null +++ b/README.md @@ -0,0 +1,139 @@ +--- +tags: +- hindi +- devanagari +- brahmi +- qwen2.5 +- transformers +- safetensors +- edge-ai +- india +- llm +- pytorch +license: apache-2.0 +language: +- hi +- en +pipeline_tag: text-generation +library_name: transformers +--- + +

+ + + + + +

+ +

ЁЯЗоЁЯЗ│ Bharat-Tiny-LLM v2 (PyTorch / Transformers Base)

+

1.5B Parameter Open-Weights Indic LLM featuring Brahmi Token Injection for 33.8% token compression and +36% faster Hindi inference.

+ +--- + +![Bharat-Tiny-LLM v2 Technical Infographic](bharat_tiny_llm_v2_tech_infographic.jpg) + +--- + +## ЁЯОп Model Overview + +**Bharat-Tiny-LLM v2** is an open-weights 1.5B parameter language model built on top of Qwen2.5-1.5B, optimized specifically for Hindi and Hinglish text generation. + +By introducing **Brahmi Token Injection** тАФ a technique that surgically injects 300 Devanagari subword tokens into the model's vocabulary тАФ Bharat-Tiny-LLM v2 eliminates the severe "Token Tax" imposed by standard English-centric tokenizers on Indian scripts. + +This repository contains the **unquantized PyTorch / HuggingFace Transformers open weights**, compatible with Linux, Windows, CUDA GPUs, vLLM, TGI, and Google Colab. + +--- + +## ЁЯОи Architectural Pipeline & Training Methodology + +![Brahmi Token Injection Architecture](bharat_tiny_llm_v2_pipeline_diagram.jpg) + +### Interactive Training Pipeline Flowchart + +```mermaid +flowchart LR + A["Raw Hindi Corpus"] --> B["Brahmi Subword Mining
(Top 300 Devanagari Tokens)"] + B --> C["Tokenizer Vocabulary Expansion
(151,936 тЮФ 152,236)"] + C --> D["Stage 1: Embedding Alignment
(Freeze Backbone, Train 300 Embeddings)"] + D --> E["Stage 2: LoRA Fine-Tuning
(Rank=16 on Attention q,k,v,o proj)"] + E --> F["Fused PyTorch Base Weights
(eulogik/Bharat-Tiny-LLM-v2)"] + F --> G["Q4 Affine Quantization
(eulogik/Bharat-Tiny-LLM-v2-MLX)"] +``` + +--- + +## тЬи Key Benchmarks & Technical Advantages + +| Metric | Base Qwen2.5-1.5B | Bharat-Tiny-LLM v2 | Technical Advantage | +|:---|:---|:---|:---| +| **Tokens for 1,000 Hindi Chars** | ~950 tokens | **~630 tokens** | **33.8% Fewer Tokens (up to 58% on chat prompts)** | +| **Inference Throughput (Hindi)** | 50 tok/s | **68 tok/s** | **+36% Speed Boost** | +| **Validation Loss (Hindi Corpus)** | 2.776 | **1.837** | **52.5% Loss Reduction (Perplexity: 16.1 тЖТ 6.3)** | +| **Hardware Compatibility** | CUDA / CPU / MPS | CUDA / CPU / MPS | **Universal PyTorch / vLLM / GGUF support** | + +--- + +## ЁЯЪА Quick Start with PyTorch & Transformers + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_name = "eulogik/Bharat-Tiny-LLM-v2" + +# Load Tokenizer & Model Weights +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.float16, + device_map="auto" +) + +# Generate Hindi Text +prompt = "рднрд╛рд░рдд рдХреА рд╕рд╛рдВрд╕реНрдХреГрддрд┐рдХ рд╡рд┐рд╡рд┐рдзрддрд╛ рдХреЗ рдмрд╛рд░реЗ рдореЗрдВ рдмрддрд╛рдЗрдП:" +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + +outputs = model.generate( + **inputs, + max_new_tokens=150, + temperature=0.7, + do_sample=True +) + +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +--- + +## ЁЯУК Token Compression Benchmarks + +| Prompt (Hindi / Hinglish) | Base Qwen Tokens | Bharat-v2 Tokens | Savings | +|:---|:---|:---|:---| +| `"рдЬрд╝рд░реВрд░реА рдмрд╛рдд рд╣реИ рдХреНрдпрд╛ рдХрд░рддреЗ рд╣реЛ"` | 26 tokens | **11 tokens** | **58% Savings** | +| `"рдирдорд╕реНрддреЗ, рдЖрдк рдХреИрд╕реЗ рд╣реИрдВ?"` | 15 tokens | **7 tokens** | **53% Savings** | +| `"рднрд╛рд░рдд рдХреА рд░рд╛рдЬрдзрд╛рдиреА рдирдИ рджрд┐рд▓реНрд▓реА рд╣реИ"` | 22 tokens | **14 tokens** | **36% Savings** | +| `"bhai aaj ka weather kaisa hai?"` | 12 tokens | **8 tokens** | **33% Savings** | + +--- + +## ЁЯФЧ Model Family Repositories + +- **Apple Silicon MLX Quantized (880MB)**: [`eulogik/Bharat-Tiny-LLM-v2-MLX`](https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2-MLX) +- **GGUF / llama.cpp**: [`eulogik/Bharat-Tiny-LLM-GGUF`](https://huggingface.co/eulogik/Bharat-Tiny-LLM-GGUF) + +--- + +## ЁЯУЬ License & Citation + +Licensed under **Apache 2.0**. Free for commercial, enterprise, and research use. + +```bibtex +@misc{kishore2026brahmi, + title={Brahmi: Efficient Devanagari Token Injection for Multilingual LLMs}, + author={Gautam Kishore}, + year={2026}, + publisher={eulogik}, + howpublished={\url{https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2}} +} +``` diff --git a/bharat_tiny_llm_v2_pipeline_diagram.jpg b/bharat_tiny_llm_v2_pipeline_diagram.jpg new file mode 100644 index 0000000..b0a5ad9 --- /dev/null +++ b/bharat_tiny_llm_v2_pipeline_diagram.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1b0ab0b62a4fdfdf792d4fac553dab6318c82ae96636714651228668e2375aa +size 574022 diff --git a/bharat_tiny_llm_v2_tech_infographic.jpg b/bharat_tiny_llm_v2_tech_infographic.jpg new file mode 100644 index 0000000..48d764e --- /dev/null +++ b/bharat_tiny_llm_v2_tech_infographic.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc4a51d9c4803c7ff451d4b042e249cae6b85734f45166e7f4aa2a0e5c40e12 +size 601107 diff --git a/brahmi_continued_pretrain.ipynb b/brahmi_continued_pretrain.ipynb new file mode 100644 index 0000000..7369947 --- /dev/null +++ b/brahmi_continued_pretrain.ipynb @@ -0,0 +1,394 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {"id": "heading"}, + "source": [ + "# Brahmi Embedding Warmup тАФ Aggressive Round 2\n", + "\n", + "**Goal**: Train the 300 new Devanagari embeddings using ALL 435K training rows.\n", + "Round 1 (150 steps, 2000 chunks) produced garbled Hindi. This round uses:\n", + "- Full corpus (435K rows тЖТ ~200K+ chunks)\n", + "- Higher LR (1e-3)\n", + "- Cosine schedule\n", + "- Only chunks containing new tokens (efficient training)\n", + "\n", + "**Hardware**: T4 GPU (Colab free tier). ~2-3 hours.\n", + "\n", + "**Before starting**: Paste your HF_TOKEN below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "install"}, + "outputs": [], + "source": [ + "!pip install -q transformers torch datasets accelerate huggingface_hub\n", + "\n", + "import json, os, math, time, random\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader\n", + "from transformers import AutoTokenizer, AutoModelForCausalLM\n", + "\n", + "HF_TOKEN = \"hf_YOUR_TOKEN_HERE\"\n", + "\n", + "# Mount Google Drive for persistence across sessions\n", + "from google.colab import drive\n", + "drive.mount('/content/drive')\n", + "DRIVE_DIR = '/content/drive/MyDrive/brahmi_training'\n", + "os.makedirs(DRIVE_DIR, exist_ok=True)\n", + "print(f'Saving to {DRIVE_DIR}')\n", + "\n", + "from huggingface_hub import login\n", + "login(token=HF_TOKEN)\n", + "\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "print(f'Device: {device}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "load"}, + "outputs": [], + "source": [ + "MODEL_ID = 'eulogik/Bharat-Tiny-LLM-v2'\n", + "\n", + "tok = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)\n", + "if tok.pad_token is None:\n", + " tok.pad_token = tok.eos_token\n", + "\n", + "model = AutoModelForCausalLM.from_pretrained(\n", + " MODEL_ID,\n", + " torch_dtype=torch.bfloat16,\n", + " device_map='auto',\n", + " token=HF_TOKEN,\n", + ")\n", + "\n", + "new_ids = sorted(\n", + " tid for tid, t in tok.added_tokens_decoder.items()\n", + " if not str(t).startswith('<') and not getattr(t, 'special', False)\n", + ")\n", + "print(f'Found {len(new_ids)} new tokens (IDs {new_ids[0]}тАУ{new_ids[-1]})')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "freeze"}, + "outputs": [], + "source": [ + "for p in model.parameters():\n", + " p.requires_grad = False\n", + "model.model.embed_tokens.weight.requires_grad_(True)\n", + "model.lm_head.weight.requires_grad_(True)\n", + "\n", + "class LearnableRows(nn.Module):\n", + " def __init__(self, embed, lmhead):\n", + " super().__init__()\n", + " self.embed = nn.Parameter(embed)\n", + " self.lmhead = nn.Parameter(lmhead)\n", + "\n", + "hidden = model.config.hidden_size\n", + "embed_w = model.model.embed_tokens.weight\n", + "lmhead_w = model.lm_head.weight\n", + "\n", + "learner = LearnableRows(\n", + " embed_w.data[new_ids].clone(),\n", + " lmhead_w.data[new_ids].clone()\n", + ").to(device)\n", + "\n", + "n_params = len(new_ids) * hidden * 2\n", + "print(f'Training {n_params:,} params (0.04% of {sum(p.numel() for p in model.parameters()):,})')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "data_load"}, + "outputs": [], + "source": [ + "import requests, gzip, shutil\n", + "\n", + "DATA_URL = 'https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2/resolve/main/train_gold_v3.jsonl.gz'\n", + "DATA_FILE = '/content/train_gold_v3.jsonl'\n", + "\n", + "if not os.path.exists(DATA_FILE):\n", + " print('Downloading...')\n", + " r = requests.get(DATA_URL, stream=True,\n", + " headers={'Authorization': f'Bearer {HF_TOKEN}'})\n", + " r.raise_for_status()\n", + " with open('/content/data.gz', 'wb') as f:\n", + " shutil.copyfileobj(r.raw, f)\n", + " with gzip.open('/content/data.gz', 'rb') as gz, open(DATA_FILE, 'wb') as f:\n", + " shutil.copyfileobj(gz, f)\n", + " os.remove('/content/data.gz')\n", + " print(f'Ready! {os.path.getsize(DATA_FILE) / 1e6:.0f} MB')\n", + "\n", + "from datasets import load_dataset\n", + "dataset = load_dataset('text', data_files=DATA_FILE, split='train')\n", + "print(f'Loaded {len(dataset)} rows')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "tokenize"}, + "outputs": [], + "source": [ + "CHUNKS_FILE = os.path.join(DRIVE_DIR, 'all_chunks.json')\n", + "\n", + "if os.path.exists(CHUNKS_FILE):\n", + " print('Loading cached chunks from Drive...')\n", + " with open(CHUNKS_FILE) as f:\n", + " all_chunks = json.load(f)\n", + " random.shuffle(all_chunks)\n", + " print(f'Loaded {len(all_chunks)} chunks')\n", + "else:\n", + " new_id_set = set(new_ids)\n", + " all_chunks = []\n", + " skipped = 0\n", + "\n", + " for i, example in enumerate(dataset):\n", + " text = example.get('text', '')\n", + " if isinstance(text, str) and text.startswith('{'):\n", + " try:\n", + " data = json.loads(text)\n", + " text = ' '.join(m['content'] for m in data.get('messages', []))\n", + " except:\n", + " pass\n", + " ids = tok.encode(text)\n", + " for j in range(0, len(ids), 256):\n", + " chunk = ids[j:j+256]\n", + " if len(chunk) >= 10:\n", + " if any(tid in new_id_set for tid in chunk):\n", + " all_chunks.append(chunk)\n", + " else:\n", + " skipped += 1\n", + " if i % 50000 == 0 and i > 0:\n", + " print(f' Processed {i}/{len(dataset)} rows...')\n", + "\n", + " random.shuffle(all_chunks)\n", + " with open(CHUNKS_FILE, 'w') as f:\n", + " json.dump(all_chunks, f)\n", + " print(f'\\nKept {len(all_chunks)} chunks (skipped {skipped})')\n", + " print(f'Saved to Drive')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "dataloader"}, + "outputs": [], + "source": [ + "def collate(batch):\n", + " mx = max(len(x) for x in batch)\n", + " pad = torch.zeros(len(batch), mx, dtype=torch.long)\n", + " for i, x in enumerate(batch):\n", + " pad[i, :len(x)] = torch.tensor(x, dtype=torch.long)\n", + " return pad.to(device)\n", + "\n", + "split = int(len(all_chunks) * 0.95)\n", + "train_chunks = all_chunks[:split]\n", + "val_chunks = all_chunks[split:]\n", + "\n", + "BATCH_SIZE = 4\n", + "train_loader = DataLoader(train_chunks, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collate)\n", + "val_loader = DataLoader(val_chunks, batch_size=BATCH_SIZE, collate_fn=collate)\n", + "\n", + "print(f'Train: {len(train_chunks)} chunks ({len(train_loader)} batches)')\n", + "print(f'Val: {len(val_chunks)} chunks ({len(val_loader)} batches)')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "init_val"}, + "outputs": [], + "source": [ + "NUM_STEPS = 3000\n", + "LR = 1e-3\n", + "CKPT_DIR = os.path.join(DRIVE_DIR, 'checkpoints')\n", + "os.makedirs(CKPT_DIR, exist_ok=True)\n", + "\n", + "optim = torch.optim.AdamW(learner.parameters(), lr=LR, weight_decay=0.01)\n", + "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n", + " optim, T_max=NUM_STEPS, eta_min=1e-5\n", + ")\n", + "\n", + "# Resume from checkpoint if exists\n", + "step = 0\n", + "best_loss = float('inf')\n", + "init_loss = None\n", + "resume_path = os.path.join(CKPT_DIR, 'state.json')\n", + "learner_path = os.path.join(CKPT_DIR, 'learner.pt')\n", + "\n", + "if os.path.exists(resume_path) and os.path.exists(learner_path):\n", + " with open(resume_path) as f:\n", + " state = json.load(f)\n", + " step = state['step']\n", + " best_loss = state['best_loss']\n", + " init_loss = state['init_loss']\n", + " learner.load_state_dict(torch.load(learner_path, map_location=device))\n", + " # Fast-forward scheduler\n", + " for _ in range(step):\n", + " scheduler.step()\n", + " print(f'Resumed from step {step}, best loss {best_loss:.4f}')\n", + "else:\n", + " # Initial validation loss\n", + " model.eval()\n", + " init_losses = []\n", + " with torch.no_grad():\n", + " for i, batch in enumerate(val_loader):\n", + " init_losses.append(model(batch, labels=batch).loss.item())\n", + " if i >= 20: break\n", + " init_loss = np.mean(init_losses)\n", + " best_loss = init_loss\n", + " print(f'Initial val loss: {init_loss:.4f} (PPL: {math.exp(init_loss):.1f})')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "train_loop"}, + "outputs": [], + "source": [ + "model.train()\n", + "start_t = time.time()\n", + "log_every = 100\n", + "\n", + "for epoch in range(10):\n", + " for batch in train_loader:\n", + " if step >= NUM_STEPS:\n", + " break\n", + "\n", + " # Copy learner тЖТ model\n", + " with torch.no_grad():\n", + " embed_w.data[new_ids] = learner.embed.data.to(embed_w.device)\n", + " lmhead_w.data[new_ids] = learner.lmhead.data.to(lmhead_w.device)\n", + "\n", + " out = model(batch, labels=batch)\n", + " loss = out.loss\n", + " loss.backward()\n", + "\n", + " # Copy grads тЖТ learner\n", + " with torch.no_grad():\n", + " learner.embed.grad = embed_w.grad[new_ids].clone().to(learner.embed.device)\n", + " learner.lmhead.grad = lmhead_w.grad[new_ids].clone().to(learner.lmhead.device)\n", + "\n", + " torch.nn.utils.clip_grad_norm_(learner.parameters(), 1.0)\n", + " optim.step()\n", + " scheduler.step()\n", + " optim.zero_grad()\n", + " model.zero_grad()\n", + " step += 1\n", + "\n", + " if step % log_every == 0:\n", + " model.eval()\n", + " vlosses = []\n", + " with torch.no_grad():\n", + " for vb in val_loader:\n", + " vlosses.append(model(vb, labels=vb).loss.item())\n", + " if len(vlosses) >= 20: break\n", + " avg_vloss = np.mean(vlosses)\n", + " impr = (init_loss - avg_vloss) / init_loss * 100\n", + " lr_now = scheduler.get_last_lr()[0]\n", + " elapsed = time.time() - start_t\n", + " print(f'Step {step}/{NUM_STEPS} | '\n", + " f'Tr:{loss.item():.4f} | '\n", + " f'Val:{avg_vloss:.4f} (PPL:{math.exp(avg_vloss):.1f}) | '\n", + " f'{impr:+.1f}% | '\n", + " f'LR:{lr_now:.2e} | '\n", + " f'{step/elapsed:.2f} it/s')\n", + " if avg_vloss < best_loss:\n", + " best_loss = avg_vloss\n", + " with torch.no_grad():\n", + " embed_w.data[new_ids] = learner.embed.data.to(embed_w.device)\n", + " lmhead_w.data[new_ids] = learner.lmhead.data.to(lmhead_w.device)\n", + " model.save_pretrained('brahmi-best')\n", + " tok.save_pretrained('brahmi-best')\n", + " # Save checkpoint for resume\n", + " torch.save(learner.state_dict(), learner_path)\n", + " with open(resume_path, 'w') as f:\n", + " json.dump({'step': step, 'best_loss': best_loss, 'init_loss': init_loss}, f)\n", + " model.train()\n", + "\n", + " if step >= NUM_STEPS:\n", + " break\n", + "\n", + "elapsed = time.time() - start_t\n", + "final_impr = (init_loss - best_loss) / init_loss * 100\n", + "print(f'\\nDone! {step} steps in {elapsed:.0f}s ({step/elapsed:.2f} it/s)')\n", + "print(f'Init loss: {init_loss:.4f} -> Best: {best_loss:.4f} ({final_impr:.1f}% improvement)')\n", + "print(f'Init PPL: {math.exp(init_loss):.1f} -> Best PPL: {math.exp(best_loss):.1f}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "generate"}, + "outputs": [], + "source": [ + "prompts = [\n", + " 'рдореИрдВ рдЖрдкрдХреЛ рдмрддрд╛рдирд╛ рдЪрд╛рд╣рддрд╛ рд╣реВрдБ рдХрд┐',\n", + " 'рднрд╛рд░рдд рдХреА рд░рд╛рдЬрдзрд╛рдиреА',\n", + " 'рдирдорд╕реНрддреЗ, рдЖрдк рдХреИрд╕реЗ рд╣реИрдВ? рдореИрдВ',\n", + "]\n", + "\n", + "for p in prompts:\n", + " inputs = tok(p, return_tensors='pt').to(model.device)\n", + " out = model.generate(\n", + " **inputs,\n", + " max_new_tokens=40,\n", + " temperature=0.3,\n", + " top_p=0.85,\n", + " repetition_penalty=1.25,\n", + " do_sample=True,\n", + " )\n", + " gen = tok.decode(out[0][inputs.input_ids.shape[-1]:],\n", + " skip_special_tokens=True)\n", + " print(f'Prompt: {p}')\n", + " print(f' -> {gen}')\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "upload"}, + "outputs": [], + "source": [ + "from huggingface_hub import HfApi\n", + "\n", + "with torch.no_grad():\n", + " embed_w.data[new_ids] = learner.embed.data.to(embed_w.device)\n", + " lmhead_w.data[new_ids] = learner.lmhead.data.to(lmhead_w.device)\n", + "\n", + "model.save_pretrained('brahmi-trained')\n", + "tok.save_pretrained('brahmi-trained')\n", + "print('Saved locally')\n", + "\n", + "api = HfApi(token=HF_TOKEN)\n", + "api.upload_folder(\n", + " folder_path='brahmi-trained',\n", + " repo_id='eulogik/Bharat-Tiny-LLM-v2',\n", + " repo_type='model',\n", + " commit_message=f'continued_pretrain_round2: {final_impr:.1f}% val loss improvement',\n", + ")\n", + "print('Uploaded to HF!')\n", + "print('https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2')" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": {"provenance": []}, + "kernelspec": {"display_name": "Python 3", "name": "python3"}, + "language_info": {"name": "python"} + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/brahmi_lora_qlora.ipynb b/brahmi_lora_qlora.ipynb new file mode 100644 index 0000000..7954306 --- /dev/null +++ b/brahmi_lora_qlora.ipynb @@ -0,0 +1,342 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {"id": "heading"}, + "source": [ + "# Bharat-Tiny-LLM v2: QLoRA Fine-tuning\n", + "\n", + "**Goal**: Fine-tune Qwen2.5-1.5B (with 300 new Devanagari tokens) using QLoRA.\n", + "This teaches the model to USE the new tokens for coherent Hindi generation.\n", + "\n", + "**Hardware**: T4 GPU (Colab free). ~1 hour for 20K chunks, 2 epochs.\n", + "\n", + "**DO NOT run cells manually** тАФ use Runtime тЖТ Run all." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "setup"}, + "outputs": [], + "source": [ + "!pip install -q transformers torch datasets accelerate huggingface_hub peft bitsandbytes\n", + "\n", + "import json, os, math, time, random, requests, gzip, shutil\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader, Dataset\n", + "from transformers import (\n", + " AutoTokenizer, AutoModelForCausalLM,\n", + " BitsAndBytesConfig, TrainingArguments, Trainer\n", + ")\n", + "from peft import LoraConfig, get_peft_model, TaskType, prepare_model_for_kbit_training\n", + "from huggingface_hub import login, HfApi\n", + "from google.colab import drive\n", + "from datasets import load_dataset\n", + "\n", + "# === CONFIG ===\n", + "HF_TOKEN = \"hf_YOUR_TOKEN_HERE\"\n", + "MODEL_ID = 'eulogik/Bharat-Tiny-LLM-v2'\n", + "\n", + "drive.mount('/content/drive')\n", + "DRIVE_DIR = '/content/drive/MyDrive/brahmi_lora'\n", + "os.makedirs(DRIVE_DIR, exist_ok=True)\n", + "\n", + "login(token=HF_TOKEN)\n", + "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", + "print(f'Device: {device}')\n", + "print(f'Drive: {DRIVE_DIR}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "load_model"}, + "outputs": [], + "source": [ + "bnb_config = BitsAndBytesConfig(\n", + " load_in_4bit=True,\n", + " bnb_4bit_quant_type='nf4',\n", + " bnb_4bit_compute_dtype=torch.bfloat16,\n", + " bnb_4bit_use_double_quant=True,\n", + ")\n", + "\n", + "tok = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)\n", + "if tok.pad_token is None:\n", + " tok.pad_token = tok.eos_token\n", + "\n", + "model = AutoModelForCausalLM.from_pretrained(\n", + " MODEL_ID,\n", + " quantization_config=bnb_config,\n", + " device_map='auto',\n", + " token=HF_TOKEN,\n", + ")\n", + "\n", + "model = prepare_model_for_kbit_training(model)\n", + "\n", + "new_ids = sorted(\n", + " tid for tid, t in tok.added_tokens_decoder.items()\n", + " if not str(t).startswith('<') and not getattr(t, 'special', False)\n", + ")\n", + "print(f'Found {len(new_ids)} new tokens')\n", + "print(f'Params: {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "lora"}, + "outputs": [], + "source": [ + "lora_config = LoraConfig(\n", + " task_type=TaskType.CAUSAL_LM,\n", + " r=16,\n", + " lora_alpha=32,\n", + " lora_dropout=0.05,\n", + " target_modules=['q_proj', 'k_proj', 'v_proj', 'o_proj'],\n", + " bias='none',\n", + ")\n", + "\n", + "model = get_peft_model(model, lora_config)\n", + "model.print_trainable_parameters()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "download_data"}, + "outputs": [], + "source": [ + "DATA_URL = 'https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2/resolve/main/train_gold_v3.jsonl.gz'\n", + "DATA_FILE = '/content/train_gold_v3.jsonl'\n", + "\n", + "if not os.path.exists(DATA_FILE):\n", + " print('Downloading...')\n", + " r = requests.get(DATA_URL, stream=True,\n", + " headers={'Authorization': f'Bearer {HF_TOKEN}'})\n", + " r.raise_for_status()\n", + " with open('/content/data.gz', 'wb') as f:\n", + " shutil.copyfileobj(r.raw, f)\n", + " with gzip.open('/content/data.gz', 'rb') as gz, open(DATA_FILE, 'wb') as f:\n", + " shutil.copyfileobj(gz, f)\n", + " os.remove('/content/data.gz')\n", + "\n", + "dataset = load_dataset('text', data_files=DATA_FILE, split='train')\n", + "print(f'Loaded {len(dataset)} rows')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "tokenize"}, + "outputs": [], + "source": [ + "CHUNKS_FILE = os.path.join(DRIVE_DIR, 'lora_chunks.json')\n", + "\n", + "if os.path.exists(CHUNKS_FILE):\n", + " with open(CHUNKS_FILE) as f:\n", + " all_chunks = json.load(f)\n", + " print(f'Loaded {len(all_chunks)} chunks from Drive')\n", + "else:\n", + " all_chunks = []\n", + " for i, example in enumerate(dataset):\n", + " text = example.get('text', '')\n", + " if isinstance(text, str) and text.startswith('{'):\n", + " try:\n", + " data = json.loads(text)\n", + " text = ' '.join(m['content'] for m in data.get('messages', []))\n", + " except:\n", + " pass\n", + " ids = tok.encode(text)\n", + " for j in range(0, len(ids), 512):\n", + " chunk = ids[j:j+512]\n", + " if len(chunk) >= 10:\n", + " all_chunks.append(chunk)\n", + " if i % 50000 == 0 and i > 0:\n", + " print(f' Processed {i}/{len(dataset)} rows...')\n", + "\n", + " random.shuffle(all_chunks)\n", + " if len(all_chunks) > 20000:\n", + " all_chunks = all_chunks[:20000]\n", + " with open(CHUNKS_FILE, 'w') as f:\n", + " json.dump(all_chunks, f)\n", + " print(f'Total chunks: {len(all_chunks)} (saved to Drive)')\n", + "\n", + "split = int(len(all_chunks) * 0.95)\n", + "train_chunks = all_chunks[:split]\n", + "val_chunks = all_chunks[split:]\n", + "print(f'Train: {len(train_chunks)}, Val: {len(val_chunks)}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "dataset"}, + "outputs": [], + "source": [ + "class ChunkDataset(Dataset):\n", + " def __init__(self, chunks, max_len=512):\n", + " self.chunks = chunks\n", + " self.max_len = max_len\n", + "\n", + " def __len__(self):\n", + " return len(self.chunks)\n", + "\n", + " def __getitem__(self, idx):\n", + " ids = self.chunks[idx][:self.max_len]\n", + " padded = ids + [tok.pad_token_id] * (self.max_len - len(ids))\n", + " mask = [1] * len(ids) + [0] * (self.max_len - len(ids))\n", + " return {\n", + " 'input_ids': torch.tensor(padded, dtype=torch.long),\n", + " 'attention_mask': torch.tensor(mask, dtype=torch.long),\n", + " 'labels': torch.tensor(padded, dtype=torch.long),\n", + " }\n", + "\n", + "train_dataset = ChunkDataset(train_chunks)\n", + "val_dataset = ChunkDataset(val_chunks)\n", + "print(f'Train: {len(train_dataset)}, Val: {len(val_dataset)}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "train"}, + "outputs": [], + "source": [ + "CKPT_DIR = os.path.join(DRIVE_DIR, 'checkpoints')\n", + "os.makedirs(CKPT_DIR, exist_ok=True)\n", + "\n", + "training_args = TrainingArguments(\n", + " output_dir=CKPT_DIR,\n", + " num_train_epochs=2,\n", + " per_device_train_batch_size=4,\n", + " per_device_eval_batch_size=4,\n", + " gradient_accumulation_steps=4,\n", + " learning_rate=2e-4,\n", + " weight_decay=0.01,\n", + " warmup_ratio=0.1,\n", + " lr_scheduler_type='cosine',\n", + " logging_steps=25,\n", + " eval_strategy='steps',\n", + " eval_steps=100,\n", + " save_strategy='steps',\n", + " save_steps=100,\n", + " save_total_limit=3,\n", + " load_best_model_at_end=True,\n", + " metric_for_best_model='eval_loss',\n", + " greater_is_better=False,\n", + " bf16=True,\n", + " gradient_checkpointing=True,\n", + " gradient_checkpointing_kwargs={'use_reentrant': False},\n", + " optim='paged_adamw_8bit',\n", + " max_grad_norm=1.0,\n", + " report_to='none',\n", + ")\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + " eval_dataset=val_dataset,\n", + ")\n", + "\n", + "# Resume from last checkpoint if exists\n", + "checkpoints = [d for d in os.listdir(CKPT_DIR) if d.startswith('checkpoint-')]\n", + "resume = None\n", + "if checkpoints:\n", + " latest = max(checkpoints, key=lambda x: int(x.split('-')[1]))\n", + " resume = os.path.join(CKPT_DIR, latest)\n", + " print(f'Resuming from {latest}')\n", + "\n", + "stats = training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps\n", + "print(f'Effective batch size: {stats}')\n", + "print('Starting training...')\n", + "trainer.train(resume_from_checkpoint=resume)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "save_upload"}, + "outputs": [], + "source": [ + "# Save LoRA adapter to Drive and HF\n", + "ADAPTER_DIR = os.path.join(DRIVE_DIR, 'lora_adapter')\n", + "model.save_pretrained(ADAPTER_DIR)\n", + "tok.save_pretrained(ADAPTER_DIR)\n", + "print(f'LoRA saved to {ADAPTER_DIR}')\n", + "\n", + "api = HfApi(token=HF_TOKEN)\n", + "api.upload_folder(\n", + " folder_path=ADAPTER_DIR,\n", + " repo_id='eulogik/Bharat-Tiny-LLM-v2-LoRA',\n", + " repo_type='model',\n", + " commit_message='LoRA: 2 epochs, rank=16, 2e-4',\n", + ")\n", + "print('Uploaded LoRA to HF!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "merge"}, + "outputs": [], + "source": [ + "# Merge LoRA into base and upload\n", + "print('Merging LoRA into base model...')\n", + "merged = model.merge_and_unload()\n", + "\n", + "MERGED_DIR = os.path.join(DRIVE_DIR, 'merged')\n", + "merged.save_pretrained(MERGED_DIR)\n", + "tok.save_pretrained(MERGED_DIR)\n", + "print(f'Merged model saved to {MERGED_DIR}')\n", + "\n", + "api = HfApi(token=HF_TOKEN)\n", + "api.upload_folder(\n", + " folder_path=MERGED_DIR,\n", + " repo_id='eulogik/Bharat-Tiny-LLM-v2',\n", + " repo_type='model',\n", + " commit_message='LoRA merged: rank=16, 2 epochs',\n", + ")\n", + "print('Merged model uploaded to HF!')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {"id": "generate"}, + "outputs": [], + "source": [ + "prompts = [\n", + " 'рдореИрдВ рдЖрдкрдХреЛ рдмрддрд╛рдирд╛ рдЪрд╛рд╣рддрд╛ рд╣реВрдБ рдХрд┐',\n", + " 'рднрд╛рд░рдд рдХреА рд░рд╛рдЬрдзрд╛рдиреА',\n", + " 'рдирдорд╕реНрддреЗ, рдЖрдк рдХреИрд╕реЗ рд╣реИрдВ? рдореИрдВ',\n", + "]\n", + "\n", + "for p in prompts:\n", + " inputs = tok(p, return_tensors='pt').to(model.device)\n", + " out = model.generate(\n", + " **inputs,\n", + " max_new_tokens=60,\n", + " temperature=0.3,\n", + " top_p=0.85,\n", + " repetition_penalty=1.25,\n", + " do_sample=True,\n", + " )\n", + " gen = tok.decode(out[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)\n", + " print(f'Prompt: {p}')\n", + " print(f' -> {gen}\\n')" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": {"provenance": []}, + "kernelspec": {"display_name": "Python 3", "name": "python3"}, + "language_info": {"name": "python"} + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..0135df4 --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "dtype": "bfloat16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": null, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.13.1", + "use_cache": true, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 152236 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..dd9f05c --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "bos_token_id": 151643, + "do_sample": false, + "eos_token_id": 151643, + "max_new_tokens": 2048, + "transformers_version": "5.13.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..eb9d6b4 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bc64c2a6313f53c4fb88eaed1a199348f85b82616520bf1d1cfd2270b65c5b0 +size 3088388744 diff --git a/pipeline_report.json b/pipeline_report.json new file mode 100644 index 0000000..347eb6d --- /dev/null +++ b/pipeline_report.json @@ -0,0 +1,1814 @@ +{ + "tokens_added": 300, + "token_ids": [ + 151665, + 151666, + 151667, + 151668, + 151669, + 151670, + 151671, + 151672, + 151673, + 151674, + 151675, + 151676, + 151677, + 151678, + 151679, + 151680, + 151681, + 151682, + 151683, + 151684, + 151685, + 151686, + 151687, + 151688, + 151689, + 151690, + 151691, + 151692, + 151693, + 151694, + 151695, + 151696, + 151697, + 151698, + 151699, + 151700, + 151701, + 151702, + 151703, + 151704, + 151705, + 151706, + 151707, + 151708, + 151709, + 151710, + 151711, + 151712, + 151713, + 151714, + 151715, + 151716, + 151717, + 151718, + 151719, + 151720, + 151721, + 151722, + 151723, + 151724, + 151725, + 151726, + 151727, + 151728, + 151729, + 151730, + 151731, + 151732, + 151733, + 151734, + 151735, + 151736, + 151737, + 151738, + 151739, + 151740, + 151741, + 151742, + 151743, + 151744, + 151745, + 151746, + 151747, + 151748, + 151749, + 151750, + 151751, + 151752, + 151753, + 151754, + 151755, + 151756, + 151757, + 151758, + 151759, + 151760, + 151761, + 151762, + 151763, + 151764, + 151765, + 151766, + 151767, + 151768, + 151769, + 151770, + 151771, + 151772, + 151773, + 151774, + 151775, + 151776, + 151777, + 151778, + 151779, + 151780, + 151781, + 151782, + 151783, + 151784, + 151785, + 151786, + 151787, + 151788, + 151789, + 151790, + 151791, + 151792, + 151793, + 151794, + 151795, + 151796, + 151797, + 151798, + 151799, + 151800, + 151801, + 151802, + 151803, + 151804, + 151805, + 151806, + 151807, + 151808, + 151809, + 151810, + 151811, + 151812, + 151813, + 151814, + 151815, + 151816, + 151817, + 151818, + 151819, + 151820, + 151821, + 151822, + 151823, + 151824, + 151825, + 151826, + 151827, + 151828, + 151829, + 151830, + 151831, + 151832, + 151833, + 151834, + 151835, + 151836, + 151837, + 151838, + 151839, + 151840, + 151841, + 151842, + 151843, + 151844, + 151845, + 151846, + 151847, + 151848, + 151849, + 151850, + 151851, + 151852, + 151853, + 151854, + 151855, + 151856, + 151857, + 151858, + 151859, + 151860, + 151861, + 151862, + 151863, + 151864, + 151865, + 151866, + 151867, + 151868, + 151869, + 151870, + 151871, + 151872, + 151873, + 151874, + 151875, + 151876, + 151877, + 151878, + 151879, + 151880, + 151881, + 151882, + 151883, + 151884, + 151885, + 151886, + 151887, + 151888, + 151889, + 151890, + 151891, + 151892, + 151893, + 151894, + 151895, + 151896, + 151897, + 151898, + 151899, + 151900, + 151901, + 151902, + 151903, + 151904, + 151905, + 151906, + 151907, + 151908, + 151909, + 151910, + 151911, + 151912, + 151913, + 151914, + 151915, + 151916, + 151917, + 151918, + 151919, + 151920, + 151921, + 151922, + 151923, + 151924, + 151925, + 151926, + 151927, + 151928, + 151929, + 151930, + 151931, + 151932, + 151933, + 151934, + 151935, + 151936, + 151937, + 151938, + 151939, + 151940, + 151941, + 151942, + 151943, + 151944, + 151945, + 151946, + 151947, + 151948, + 151949, + 151950, + 151951, + 151952, + 151953, + 151954, + 151955, + 151956, + 151957, + 151958, + 151959, + 151960, + 151961, + 151962, + 151963, + 151964 + ], + "old_vocab_size": 151936, + "new_vocab_size": 152236, + "compression_chars_per_token_baseline": 434.975, + "compression_chars_per_token_improved": 657.316, + "compression_improvement_pct": 51.1, + "token_savings_pct": 33.8, + "init_method": "mean", + "selected_tokens": [ + { + "token": "рд╣реИ", + "saved_tokens": 2, + "frequency": 33145 + }, + { + "token": "рд╣реИрдВ", + "saved_tokens": 3, + "frequency": 11690 + }, + { + "token": "рд╣реИред", + "saved_tokens": 3, + "frequency": 6916 + }, + { + "token": "реИрдВ", + "saved_tokens": 2, + "frequency": 14416 + }, + { + "token": "рд╣реИрдВред", + "saved_tokens": 4, + "frequency": 3376 + }, + { + "token": "рдЬрд╝рд░реВрд░реА", + "saved_tokens": 7, + "frequency": 971 + }, + { + "token": "рдореЗрдВ", + "saved_tokens": 2, + "frequency": 11848 + }, + { + "token": "рдЬрд╝рд░реВрд░", + "saved_tokens": 6, + "frequency": 1292 + }, + { + "token": "рдХреНрдпрд╛", + "saved_tokens": 3, + "frequency": 5063 + }, + { + "token": "рдирд╣реАрдВ", + "saved_tokens": 3, + "frequency": 3928 + }, + { + "token": "реИрд╕реЗ", + "saved_tokens": 3, + "frequency": 3904 + }, + { + "token": "рд╝рд░реВрд░реА", + "saved_tokens": 6, + "frequency": 971 + }, + { + "token": "рдЬрд╝реНрдпрд╛рдж", + "saved_tokens": 6, + "frequency": 900 + }, + { + "token": "рд╝реНрдпрд╛рджрд╛", + "saved_tokens": 6, + "frequency": 899 + }, + { + "token": "рдЬрд╝рд░реВ", + "saved_tokens": 5, + "frequency": 1292 + }, + { + "token": "рд╝рд░реВрд░", + "saved_tokens": 5, + "frequency": 1292 + }, + { + "token": "рдХреИрд╕реЗ", + "saved_tokens": 4, + "frequency": 2002 + }, + { + "token": "реНрдпрд╛", + "saved_tokens": 2, + "frequency": 7829 + }, + { + "token": "рдкреЙрдЗрдВрдЯ", + "saved_tokens": 5, + "frequency": 1238 + }, + { + "token": "реИрдВред", + "saved_tokens": 3, + "frequency": 3376 + }, + { + "token": "рдЬрд╝", + "saved_tokens": 2, + "frequency": 7267 + }, + { + "token": "реИред", + "saved_tokens": 2, + "frequency": 6916 + }, + { + "token": "рдлреЙрд░реНрдо", + "saved_tokens": 5, + "frequency": 1092 + }, + { + "token": "рдлреЙрд░реНрдореЗ", + "saved_tokens": 6, + "frequency": 733 + }, + { + "token": "рдЬреИрд╕реЗ", + "saved_tokens": 4, + "frequency": 1585 + }, + { + "token": "рддреБрдореНрд╣рд╛", + "saved_tokens": 5, + "frequency": 1011 + }, + { + "token": "реБрдореНрд╣рд╛рд░", + "saved_tokens": 5, + "frequency": 1011 + }, + { + "token": "рд╣реВрдБ", + "saved_tokens": 4, + "frequency": 1578 + }, + { + "token": "рдХреНрдп", + "saved_tokens": 2, + "frequency": 6078 + }, + { + "token": "рд╕реНрдЯ", + "saved_tokens": 2, + "frequency": 6004 + }, + { + "token": "рддреБрдореНрд╣", + "saved_tokens": 4, + "frequency": 1494 + }, + { + "token": "рдмрд╣реБрдд", + "saved_tokens": 3, + "frequency": 2602 + }, + { + "token": "рдпреВрдЬрд╝", + "saved_tokens": 5, + "frequency": 930 + }, + { + "token": "рдЬрд╝реНрдпрд╛", + "saved_tokens": 5, + "frequency": 901 + }, + { + "token": "рд╝реНрдпрд╛рдж", + "saved_tokens": 5, + "frequency": 900 + }, + { + "token": "реЙрдЗрдВрдЯреНрд╕", + "saved_tokens": 6, + "frequency": 598 + }, + { + "token": "рдпрд╛", + "saved_tokens": 1, + "frequency": 21373 + }, + { + "token": "рдкреЙрдЗрдВрдЯреН", + "saved_tokens": 6, + "frequency": 588 + }, + { + "token": "реЙрд░реНрдо", + "saved_tokens": 4, + "frequency": 1300 + }, + { + "token": "рд╝рд░реВ", + "saved_tokens": 4, + "frequency": 1292 + }, + { + "token": "рдХреИрд╕", + "saved_tokens": 3, + "frequency": 2253 + }, + { + "token": "реЙрдЗрдВрдЯ", + "saved_tokens": 4, + "frequency": 1260 + }, + { + "token": "рдкреНрд▓рд╛рди", + "saved_tokens": 4, + "frequency": 1254 + }, + { + "token": "реЗрдВ", + "saved_tokens": 1, + "frequency": 20044 + }, + { + "token": "рдкреЙрдЗрдВ", + "saved_tokens": 4, + "frequency": 1239 + }, + { + "token": "реЙрд░реНрдореЗ", + "saved_tokens": 5, + "frequency": 782 + }, + { + "token": "реИрд╕", + "saved_tokens": 2, + "frequency": 4850 + }, + { + "token": "рдореИрдВ", + "saved_tokens": 3, + "frequency": 2154 + }, + { + "token": "реНрдпрд╛рджрд╛", + "saved_tokens": 4, + "frequency": 1184 + }, + { + "token": "рдереЛрдбрд╝рд╛", + "saved_tokens": 5, + "frequency": 751 + }, + { + "token": "реВрдБ", + "saved_tokens": 3, + "frequency": 2061 + }, + { + "token": "рдбрд╝", + "saved_tokens": 2, + "frequency": 4580 + }, + { + "token": "рдЬреИрд╕", + "saved_tokens": 3, + "frequency": 1969 + }, + { + "token": "реЙрд░реНрдореЗрд╢", + "saved_tokens": 6, + "frequency": 489 + }, + { + "token": "рдлреЙрд░реН", + "saved_tokens": 4, + "frequency": 1093 + }, + { + "token": "рд╣реАрдВ", + "saved_tokens": 2, + "frequency": 4264 + }, + { + "token": "рдХрд░рдиреЗ", + "saved_tokens": 3, + "frequency": 1866 + }, + { + "token": "рдХреН", + "saved_tokens": 1, + "frequency": 16562 + }, + { + "token": "рд░реВрд░реА", + "saved_tokens": 4, + "frequency": 1033 + }, + { + "token": "рдХрд░рдирд╛", + "saved_tokens": 3, + "frequency": 1822 + }, + { + "token": "рд╕реЗ", + "saved_tokens": 1, + "frequency": 16360 + }, + { + "token": "рд╣реЛрддрд╛", + "saved_tokens": 3, + "frequency": 1807 + }, + { + "token": "реБрдореНрд╣рд╛", + "saved_tokens": 4, + "frequency": 1011 + }, + { + "token": "рдореНрд╣рд╛рд░", + "saved_tokens": 4, + "frequency": 1011 + }, + { + "token": "рдХреИрдЬрд╝реБрдЕ", + "saved_tokens": 7, + "frequency": 327 + }, + { + "token": "реИрдЬрд╝реБрдЕрд▓", + "saved_tokens": 7, + "frequency": 327 + }, + { + "token": "рдереЛрдбрд╝", + "saved_tokens": 4, + "frequency": 988 + }, + { + "token": "рдирд╣реА", + "saved_tokens": 2, + "frequency": 3929 + }, + { + "token": "рдХрд░рди", + "saved_tokens": 2, + "frequency": 3873 + }, + { + "token": "реВрдЬрд╝", + "saved_tokens": 4, + "frequency": 966 + }, + { + "token": "рдЪрд╛рд╣рд┐рдП", + "saved_tokens": 4, + "frequency": 942 + }, + { + "token": "рдХрд░рддреЗ", + "saved_tokens": 3, + "frequency": 1673 + }, + { + "token": "рдмрд┐рд▓реНрдХреБ", + "saved_tokens": 5, + "frequency": 602 + }, + { + "token": "рд┐рд▓реНрдХреБрд▓", + "saved_tokens": 5, + "frequency": 602 + }, + { + "token": "рд┐рдпрд╛", + "saved_tokens": 2, + "frequency": 3750 + }, + { + "token": "реЙрдЗрдВрдЯреН", + "saved_tokens": 5, + "frequency": 599 + }, + { + "token": "рдХрд░", + "saved_tokens": 1, + "frequency": 14963 + }, + { + "token": "рд▓рд┐рдП", + "saved_tokens": 2, + "frequency": 3722 + }, + { + "token": "рдЗрдВрдбрд┐рдп", + "saved_tokens": 4, + "frequency": 929 + }, + { + "token": "рдореЗ", + "saved_tokens": 1, + "frequency": 14729 + }, + { + "token": "рдЬрд╝реНрдп", + "saved_tokens": 4, + "frequency": 920 + }, + { + "token": "рд╣реЛрдд", + "saved_tokens": 2, + "frequency": 3646 + }, + { + "token": "рд╡рдкреВрд░реНрдг", + "saved_tokens": 6, + "frequency": 405 + }, + { + "token": "рд╣рддреНрд╡рдкреВ", + "saved_tokens": 6, + "frequency": 403 + }, + { + "token": "рддреНрд╡рдкреВрд░", + "saved_tokens": 6, + "frequency": 402 + }, + { + "token": "реНрд╡рдкреВрд░реН", + "saved_tokens": 6, + "frequency": 402 + }, + { + "token": "рд╝реНрдпрд╛", + "saved_tokens": 4, + "frequency": 904 + }, + { + "token": "реНрдпреВ", + "saved_tokens": 3, + "frequency": 1600 + }, + { + "token": "рдмрдврд╝рд┐рдпрд╛", + "saved_tokens": 6, + "frequency": 399 + }, + { + "token": "рд╣рд╛рдБ", + "saved_tokens": 2, + "frequency": 3582 + }, + { + "token": "рдХреЗ", + "saved_tokens": 1, + "frequency": 14303 + }, + { + "token": "реЙрд░реН", + "saved_tokens": 3, + "frequency": 1573 + }, + { + "token": "рдХрд╛", + "saved_tokens": 1, + "frequency": 13939 + }, + { + "token": "рдХреЙрдиреНрдЯреЗ", + "saved_tokens": 6, + "frequency": 385 + }, + { + "token": "рдбрд╝рд╛", + "saved_tokens": 3, + "frequency": 1531 + }, + { + "token": "рдореНрд╣рд╛рд░реА", + "saved_tokens": 5, + "frequency": 549 + }, + { + "token": "рд╡реИрд▓реНрдпреВ", + "saved_tokens": 7, + "frequency": 280 + }, + { + "token": "реЙрдиреНрдЯреЗрдХ", + "saved_tokens": 6, + "frequency": 381 + }, + { + "token": "рд╛рд░", + "saved_tokens": 1, + "frequency": 13648 + }, + { + "token": "реНрдп", + "saved_tokens": 1, + "frequency": 13535 + }, + { + "token": "рдЬрд╝рд░", + "saved_tokens": 3, + "frequency": 1499 + }, + { + "token": "рддреБрдореН", + "saved_tokens": 3, + "frequency": 1495 + }, + { + "token": "рдЗрдВрдбрд┐рдпрд╛", + "saved_tokens": 5, + "frequency": 538 + }, + { + "token": "реБрдореНрд╣", + "saved_tokens": 3, + "frequency": 1494 + }, + { + "token": "рдкреНрд▓рд╛", + "saved_tokens": 3, + "frequency": 1485 + }, + { + "token": "рдпрд╛рд░", + "saved_tokens": 2, + "frequency": 3333 + }, + { + "token": "рд░реВрд░", + "saved_tokens": 3, + "frequency": 1470 + }, + { + "token": "рд░реН", + "saved_tokens": 1, + "frequency": 13216 + }, + { + "token": "реВрд░", + "saved_tokens": 2, + "frequency": 3293 + }, + { + "token": "рдЪреЗрдХрд▓рд┐рд╕", + "saved_tokens": 5, + "frequency": 526 + }, + { + "token": "реЗрдХрд▓рд┐рд╕реН", + "saved_tokens": 5, + "frequency": 526 + }, + { + "token": "рдХрд▓рд┐рд╕реНрдЯ", + "saved_tokens": 5, + "frequency": 526 + }, + { + "token": "рд╕реНрдЯреН", + "saved_tokens": 3, + "frequency": 1455 + }, + { + "token": "рд░реНрдореЗрд╢рди", + "saved_tokens": 5, + "frequency": 521 + }, + { + "token": "рдирд╛", + "saved_tokens": 1, + "frequency": 13016 + }, + { + "token": "рд╕рдмрд╕реЗ", + "saved_tokens": 3, + "frequency": 1445 + }, + { + "token": "реВрд░реА", + "saved_tokens": 3, + "frequency": 1442 + }, + { + "token": "рдЗрдиреНрдлреЙрд░", + "saved_tokens": 6, + "frequency": 359 + }, + { + "token": "рдиреНрдлреЙрд░реН", + "saved_tokens": 6, + "frequency": 359 + }, + { + "token": "реНрдлреЙрд░реНрдо", + "saved_tokens": 6, + "frequency": 359 + }, + { + "token": "рдХреИ", + "saved_tokens": 2, + "frequency": 3205 + }, + { + "token": "рдПрдХрджрдо", + "saved_tokens": 3, + "frequency": 1413 + }, + { + "token": "рдЕрдЪреНрдЫ", + "saved_tokens": 3, + "frequency": 1406 + }, + { + "token": "рдкреВрд░", + "saved_tokens": 3, + "frequency": 1386 + }, + { + "token": "рд░реВ", + "saved_tokens": 2, + "frequency": 3106 + }, + { + "token": "рд╕рд┐рд░реНрдл", + "saved_tokens": 4, + "frequency": 775 + }, + { + "token": "рдХреНрдпреЛрдВ", + "saved_tokens": 4, + "frequency": 771 + }, + { + "token": "рд╕реНрдЯреНрд░", + "saved_tokens": 3, + "frequency": 1370 + }, + { + "token": "рдХрд░рдд", + "saved_tokens": 2, + "frequency": 3072 + }, + { + "token": "реЛрдбрд╝рд╛", + "saved_tokens": 4, + "frequency": 767 + }, + { + "token": "рддрд╛", + "saved_tokens": 1, + "frequency": 12200 + }, + { + "token": "рддреБрдореНрд╣реЗ", + "saved_tokens": 5, + "frequency": 482 + }, + { + "token": "реБрдореНрд╣реЗрдВ", + "saved_tokens": 5, + "frequency": 481 + }, + { + "token": "рдЕрдЪреНрдЫрд╛", + "saved_tokens": 4, + "frequency": 751 + }, + { + "token": "рдмрдврд╝", + "saved_tokens": 3, + "frequency": 1329 + }, + { + "token": "рд╕реН", + "saved_tokens": 1, + "frequency": 11934 + }, + { + "token": "рд┐рд╕реНрдЯ", + "saved_tokens": 3, + "frequency": 1314 + }, + { + "token": "реАрдЬрд╝", + "saved_tokens": 3, + "frequency": 1310 + }, + { + "token": "рдХреИрдЬрд╝реБ", + "saved_tokens": 6, + "frequency": 327 + }, + { + "token": "реИрдЬрд╝реБрдЕ", + "saved_tokens": 6, + "frequency": 327 + }, + { + "token": "рдЪреАрдЬрд╝", + "saved_tokens": 4, + "frequency": 735 + }, + { + "token": "рддреИрдпрд╛рд░", + "saved_tokens": 5, + "frequency": 468 + }, + { + "token": "рд╕рдХрддреЗ", + "saved_tokens": 3, + "frequency": 1298 + }, + { + "token": "реНрдЯ", + "saved_tokens": 1, + "frequency": 11668 + }, + { + "token": "рдореИ", + "saved_tokens": 2, + "frequency": 2916 + }, + { + "token": "рд╛рди", + "saved_tokens": 1, + "frequency": 11540 + }, + { + "token": "рдХрд░реЗрдВ", + "saved_tokens": 3, + "frequency": 1282 + }, + { + "token": "рд╣реЛ", + "saved_tokens": 1, + "frequency": 11520 + }, + { + "token": "рдпреВрдЬ", + "saved_tokens": 3, + "frequency": 1276 + }, + { + "token": "рдиреЗ", + "saved_tokens": 1, + "frequency": 11388 + }, + { + "token": "реЙрдЗрдВ", + "saved_tokens": 3, + "frequency": 1264 + }, + { + "token": "реНрд▓рд╛рди", + "saved_tokens": 3, + "frequency": 1261 + }, + { + "token": "рдХрд░реВрдБрдЧрд╛", + "saved_tokens": 7, + "frequency": 231 + }, + { + "token": "рдХреЙрдиреНрдЯ", + "saved_tokens": 5, + "frequency": 452 + }, + { + "token": "реЛрдбрд╝", + "saved_tokens": 3, + "frequency": 1242 + }, + { + "token": "рдкреЙрдЗ", + "saved_tokens": 3, + "frequency": 1240 + }, + { + "token": "рд╝рд╛", + "saved_tokens": 2, + "frequency": 2768 + }, + { + "token": "рд┐рдВрдЧ", + "saved_tokens": 2, + "frequency": 2766 + }, + { + "token": "реНрд╕", + "saved_tokens": 1, + "frequency": 11009 + }, + { + "token": "рдпреВ", + "saved_tokens": 2, + "frequency": 2750 + }, + { + "token": "рдкреВрд░реНрдг", + "saved_tokens": 5, + "frequency": 440 + }, + { + "token": "рд▓рд┐рд╕реНрдЯ", + "saved_tokens": 4, + "frequency": 687 + }, + { + "token": "рдХреНрд╕", + "saved_tokens": 2, + "frequency": 2724 + }, + { + "token": "рдЬрд╛рдирдХрд╛рд░", + "saved_tokens": 5, + "frequency": 428 + }, + { + "token": "реНрдпрд╛рдж", + "saved_tokens": 3, + "frequency": 1188 + }, + { + "token": "рдмрд╣реБ", + "saved_tokens": 2, + "frequency": 2666 + }, + { + "token": "рдпрд╛рджрд╛", + "saved_tokens": 3, + "frequency": 1184 + }, + { + "token": "рдФрд░", + "saved_tokens": 1, + "frequency": 10621 + }, + { + "token": "рд╛рдирдХрд╛рд░реА", + "saved_tokens": 5, + "frequency": 424 + }, + { + "token": "рдлреЙрд░", + "saved_tokens": 3, + "frequency": 1171 + }, + { + "token": "рд╣рд┐рдВрджреА", + "saved_tokens": 4, + "frequency": 658 + }, + { + "token": "рд╣реБрдд", + "saved_tokens": 2, + "frequency": 2603 + }, + { + "token": "рдСрдирд▓рд╛рдЗрди", + "saved_tokens": 5, + "frequency": 416 + }, + { + "token": "рдХреБрдЫ", + "saved_tokens": 2, + "frequency": 2599 + }, + { + "token": "реЗрдХреНрд╕реНрдЯ", + "saved_tokens": 5, + "frequency": 415 + }, + { + "token": "рдХреЛ", + "saved_tokens": 1, + "frequency": 10340 + }, + { + "token": "рддрд░реАрдХреЗ", + "saved_tokens": 4, + "frequency": 637 + }, + { + "token": "рд╡рд╛рд▓", + "saved_tokens": 2, + "frequency": 2534 + }, + { + "token": "рд╡рдкреВрд░реН", + "saved_tokens": 5, + "frequency": 405 + }, + { + "token": "рдЪреАрдЬрд╝реЗрдВ", + "saved_tokens": 6, + "frequency": 281 + }, + { + "token": "реИрд▓реНрдпреВ", + "saved_tokens": 6, + "frequency": 280 + }, + { + "token": "рддреНрд╡рдкреВ", + "saved_tokens": 5, + "frequency": 403 + }, + { + "token": "рдорд╣рддреНрд╡рдк", + "saved_tokens": 5, + "frequency": 403 + }, + { + "token": "рддрд░реАрдХ", + "saved_tokens": 3, + "frequency": 1117 + }, + { + "token": "реНрд╡рдкреВрд░", + "saved_tokens": 5, + "frequency": 402 + }, + { + "token": "рдЯреЗрдХреНрд╕реН", + "saved_tokens": 5, + "frequency": 402 + }, + { + "token": "рдврд╝рд┐рдпрд╛", + "saved_tokens": 5, + "frequency": 400 + }, + { + "token": "рдмрдврд╝рд┐рдп", + "saved_tokens": 5, + "frequency": 399 + }, + { + "token": "рдЗрдВрдЯреНрд╕", + "saved_tokens": 4, + "frequency": 623 + }, + { + "token": "рдкреЛрд░реНрдЯ", + "saved_tokens": 4, + "frequency": 620 + }, + { + "token": "рддреЗ", + "saved_tokens": 1, + "frequency": 9876 + }, + { + "token": "реНрдЯреЗрдХреНрд╕", + "saved_tokens": 5, + "frequency": 391 + }, + { + "token": "рдмрдбрд╝", + "saved_tokens": 3, + "frequency": 1085 + }, + { + "token": "рд╕рдХрдд", + "saved_tokens": 2, + "frequency": 2441 + }, + { + "token": "реЙрдиреНрдЯреЗ", + "saved_tokens": 5, + "frequency": 387 + }, + { + "token": "рдмрд┐рд▓реНрдХ", + "saved_tokens": 4, + "frequency": 602 + }, + { + "token": "рд┐рд▓реНрдХреБ", + "saved_tokens": 4, + "frequency": 602 + }, + { + "token": "рд▓реНрдХреБрд▓", + "saved_tokens": 4, + "frequency": 602 + }, + { + "token": "рдХрд╛рдиреВрди", + "saved_tokens": 5, + "frequency": 385 + }, + { + "token": "рдЙрдиреНрд╣реЗрдВ", + "saved_tokens": 5, + "frequency": 385 + }, + { + "token": "рдЗрдВрдбрд┐рдпрди", + "saved_tokens": 5, + "frequency": 384 + }, + { + "token": "рдзреНрдпрд╛рди", + "saved_tokens": 4, + "frequency": 597 + }, + { + "token": "рдиреНрдЯреЗрдХреН", + "saved_tokens": 5, + "frequency": 381 + }, + { + "token": "рдореЗрдВрдЯ", + "saved_tokens": 3, + "frequency": 1051 + }, + { + "token": "рджреЗрдЦ", + "saved_tokens": 2, + "frequency": 2352 + }, + { + "token": "рдЯреНрд╕", + "saved_tokens": 2, + "frequency": 2345 + }, + { + "token": "рд╕реНрдЯреНрд░рдХ", + "saved_tokens": 4, + "frequency": 585 + }, + { + "token": "реНрдЯреНрд░рдХреН", + "saved_tokens": 4, + "frequency": 585 + }, + { + "token": "реВрдБрдЧрд╛", + "saved_tokens": 5, + "frequency": 370 + }, + { + "token": "рдХреЙрдиреН", + "saved_tokens": 4, + "frequency": 576 + }, + { + "token": "реНрд╣рд╛рд░", + "saved_tokens": 3, + "frequency": 1012 + }, + { + "token": "рдВрдЯреНрд╕", + "saved_tokens": 3, + "frequency": 1011 + }, + { + "token": "рдореНрд╣рд╛", + "saved_tokens": 3, + "frequency": 1011 + }, + { + "token": "рдкреНрд▓", + "saved_tokens": 2, + "frequency": 2262 + }, + { + "token": "реНрдЯреН", + "saved_tokens": 2, + "frequency": 2260 + }, + { + "token": "реЙрд░реНрдореЗрдЯ", + "saved_tokens": 6, + "frequency": 250 + }, + { + "token": "рджреЗрдЦреЛ", + "saved_tokens": 3, + "frequency": 999 + }, + { + "token": "реНрд╣реЗрдВ", + "saved_tokens": 3, + "frequency": 998 + }, + { + "token": "рдЗрдиреНрдлреЙ", + "saved_tokens": 5, + "frequency": 359 + }, + { + "token": "рдиреНрдлреЙрд░", + "saved_tokens": 5, + "frequency": 359 + }, + { + "token": "реНрдлреЙрд░реН", + "saved_tokens": 5, + "frequency": 359 + }, + { + "token": "рд╣реЛрддреА", + "saved_tokens": 3, + "frequency": 996 + }, + { + "token": "рдЬрд╝реН", + "saved_tokens": 3, + "frequency": 989 + }, + { + "token": "рдХрд░реЗрдВрдЧреЗ", + "saved_tokens": 5, + "frequency": 353 + }, + { + "token": "рддреБрдо", + "saved_tokens": 2, + "frequency": 2200 + }, + { + "token": "реНрд╣рд╛рд░реА", + "saved_tokens": 4, + "frequency": 550 + }, + { + "token": "рдХрд╛рд░", + "saved_tokens": 2, + "frequency": 2184 + }, + { + "token": "рдЬрд╝реБрдЕрд▓", + "saved_tokens": 5, + "frequency": 349 + }, + { + "token": "рдкрдбрд╝рддрд╛", + "saved_tokens": 5, + "frequency": 347 + }, + { + "token": "рдВрдбрд┐рдпрд╛", + "saved_tokens": 4, + "frequency": 541 + }, + { + "token": "рдкреН", + "saved_tokens": 1, + "frequency": 8617 + }, + { + "token": "рдЗрдВрдбрд┐", + "saved_tokens": 3, + "frequency": 957 + }, + { + "token": "рдХрд┐", + "saved_tokens": 1, + "frequency": 8611 + }, + { + "token": "реНрдпреЛрдВ", + "saved_tokens": 3, + "frequency": 951 + }, + { + "token": "рдХрд░рддрд╛", + "saved_tokens": 3, + "frequency": 951 + }, + { + "token": "рд░рдиреЗ", + "saved_tokens": 2, + "frequency": 2139 + }, + { + "token": "рд╛рдирд╛", + "saved_tokens": 2, + "frequency": 2134 + }, + { + "token": "рдЪрд╛рд╣рд┐", + "saved_tokens": 3, + "frequency": 943 + }, + { + "token": "рд╛рд╣рд┐рдП", + "saved_tokens": 3, + "frequency": 942 + }, + { + "token": "рд░реВрдБрдЧрд╛", + "saved_tokens": 6, + "frequency": 235 + }, + { + "token": "рдврд╝", + "saved_tokens": 2, + "frequency": 2112 + }, + { + "token": "рдкреВрд░реН", + "saved_tokens": 4, + "frequency": 528 + }, + { + "token": "рдВрдбрд┐рдп", + "saved_tokens": 3, + "frequency": 937 + }, + { + "token": "реЗрдХреНрдЯ", + "saved_tokens": 3, + "frequency": 937 + }, + { + "token": "рд╛рд░реА", + "saved_tokens": 2, + "frequency": 2104 + }, + { + "token": "рдЪреЗрдХрд▓рд┐", + "saved_tokens": 4, + "frequency": 526 + }, + { + "token": "реЗрдХрд▓рд┐рд╕", + "saved_tokens": 4, + "frequency": 526 + }, + { + "token": "рдХрд▓рд┐рд╕реН", + "saved_tokens": 4, + "frequency": 526 + }, + { + "token": "рдЬреИ", + "saved_tokens": 2, + "frequency": 2101 + }, + { + "token": "рдмрдирд╛", + "saved_tokens": 2, + "frequency": 2097 + }, + { + "token": "рдбрд┐рдпрд╛", + "saved_tokens": 3, + "frequency": 932 + }, + { + "token": "рдмрдбрд╝рд╛", + "saved_tokens": 4, + "frequency": 523 + }, + { + "token": "реЗрдХреН", + "saved_tokens": 2, + "frequency": 2088 + }, + { + "token": "рд░реНрдореЗрд╢", + "saved_tokens": 4, + "frequency": 522 + }, + { + "token": "рдХрд░реВрдБрдЧ", + "saved_tokens": 6, + "frequency": 232 + }, + { + "token": "реНрдореЗрд╢рди", + "saved_tokens": 4, + "frequency": 521 + }, + { + "token": "рдЬрд╝рд░реВрд░рдд", + "saved_tokens": 7, + "frequency": 170 + }, + { + "token": "рд╝реНрдп", + "saved_tokens": 3, + "frequency": 924 + }, + { + "token": "рдиреНрд╣реЗрдВ", + "saved_tokens": 4, + "frequency": 517 + }, + { + "token": "рд╣реА", + "saved_tokens": 1, + "frequency": 8269 + }, + { + "token": "рдЗрдВрд╕реНрдЯреН", + "saved_tokens": 5, + "frequency": 330 + }, + { + "token": "реЗрд╕реНрдЯ", + "saved_tokens": 3, + "frequency": 915 + }, + { + "token": "рд░реЗ", + "saved_tokens": 1, + "frequency": 8188 + }, + { + "token": "рд╣рд╛", + "saved_tokens": 1, + "frequency": 8186 + }, + { + "token": "рд╣реЛрдЧрд╛", + "saved_tokens": 3, + "frequency": 909 + }, + { + "token": "рдХреИрдЬрд╝", + "saved_tokens": 5, + "frequency": 327 + }, + { + "token": "реИрдЬрд╝реБ", + "saved_tokens": 5, + "frequency": 327 + }, + { + "token": "рдмрд╛рдд", + "saved_tokens": 2, + "frequency": 2029 + }, + { + "token": "реЗрдВрдЯ", + "saved_tokens": 2, + "frequency": 2028 + }, + { + "token": "рдорд┐рд▓", + "saved_tokens": 2, + "frequency": 2026 + }, + { + "token": "реЛрдВ", + "saved_tokens": 1, + "frequency": 8055 + }, + { + "token": "рдЕрдкрди", + "saved_tokens": 2, + "frequency": 2010 + }, + { + "token": "рдкрд╣рд▓реЗ", + "saved_tokens": 3, + "frequency": 893 + }, + { + "token": "рдЯрд╛рдЗрдо", + "saved_tokens": 3, + "frequency": 891 + }, + { + "token": "рд░реНрдо", + "saved_tokens": 2, + "frequency": 1998 + }, + { + "token": "реИрдХреНрдЯ", + "saved_tokens": 4, + "frequency": 499 + }, + { + "token": "реЗрдВрдЧреЗ", + "saved_tokens": 3, + "frequency": 886 + }, + { + "token": "рд░реА", + "saved_tokens": 1, + "frequency": 7923 + }, + { + "token": "рдХреНрдпреЛ", + "saved_tokens": 3, + "frequency": 879 + }, + { + "token": "рд┐рд╕реН", + "saved_tokens": 2, + "frequency": 1965 + }, + { + "token": "рдЕрдкрдиреА", + "saved_tokens": 3, + "frequency": 871 + }, + { + "token": "рд▓реЛрдЧреЛрдВ", + "saved_tokens": 4, + "frequency": 489 + }, + { + "token": "рдЬрд╛рддрд╛", + "saved_tokens": 3, + "frequency": 866 + }, + { + "token": "рд░рдирд╛", + "saved_tokens": 2, + "frequency": 1942 + }, + { + "token": "рдкрдбрд╝", + "saved_tokens": 3, + "frequency": 863 + }, + { + "token": "рдЖрдкрдХ", + "saved_tokens": 2, + "frequency": 1941 + }, + { + "token": "реБрдореНрд╣реЗ", + "saved_tokens": 4, + "frequency": 482 + }, + { + "token": "рдХреНрд╕реНрдЯ", + "saved_tokens": 4, + "frequency": 482 + }, + { + "token": "рдореНрд╣реЗрдВ", + "saved_tokens": 4, + "frequency": 481 + }, + { + "token": "реИрдХреН", + "saved_tokens": 3, + "frequency": 855 + }, + { + "token": "рдкреЙ", + "saved_tokens": 2, + "frequency": 1915 + }, + { + "token": "рд╛рд░реН", + "saved_tokens": 2, + "frequency": 1907 + }, + { + "token": "рд░реЗрдВ", + "saved_tokens": 2, + "frequency": 1901 + }, + { + "token": "рдЗрд╕реНрддреЗрдо", + "saved_tokens": 5, + "frequency": 304 + }, + { + "token": "рд╕реНрддреЗрдорд╛", + "saved_tokens": 5, + "frequency": 304 + }, + { + "token": "реНрддреЗрдорд╛рд▓", + "saved_tokens": 5, + "frequency": 303 + }, + { + "token": "рдЕрдХреНрд╕рд░", + "saved_tokens": 4, + "frequency": 473 + }, + { + "token": "рд╣реЛрддреЗ", + "saved_tokens": 3, + "frequency": 840 + }, + { + "token": "реЗрд╢рди", + "saved_tokens": 2, + "frequency": 1888 + }, + { + "token": "рдХрд░реВрдБ", + "saved_tokens": 5, + "frequency": 301 + } + ] +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..0db5fe9 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64b15d62a2fa1dacc30d01bf9cb0dd0c04b8945eacee37422c5491d70517dd46 +size 11478613 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..df536e9 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_gold_v3.jsonl.gz b/train_gold_v3.jsonl.gz new file mode 100644 index 0000000..b83dff8 --- /dev/null +++ b/train_gold_v3.jsonl.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b30ac1430cb4a47e67bcd4e67bed9fd273b18b2bd79ac177544abedc8869d5c7 +size 127388965