初始化项目，由ModelHub XC社区提供模型
Model: eulogik/Bharat-Tiny-LLM-v2 Source: Original Platform
This commit is contained in:
ModelHub XC
342
brahmi_lora_qlora.ipynb
Normal file
342
brahmi_lora_qlora.ipynb
Normal file
@@ -0,0 +1,342 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {"id": "heading"},
|
||||
"source": [
|
||||
"# Bharat-Tiny-LLM v2: QLoRA Fine-tuning\n",
|
||||
"\n",
|
||||
"**Goal**: Fine-tune Qwen2.5-1.5B (with 300 new Devanagari tokens) using QLoRA.\n",
|
||||
"This teaches the model to USE the new tokens for coherent Hindi generation.\n",
|
||||
"\n",
|
||||
"**Hardware**: T4 GPU (Colab free). ~1 hour for 20K chunks, 2 epochs.\n",
|
||||
"\n",
|
||||
"**DO NOT run cells manually** — use Runtime → Run all."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "setup"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install -q transformers torch datasets accelerate huggingface_hub peft bitsandbytes\n",
|
||||
"\n",
|
||||
"import json, os, math, time, random, requests, gzip, shutil\n",
|
||||
"import numpy as np\n",
|
||||
"import torch\n",
|
||||
"import torch.nn as nn\n",
|
||||
"from torch.utils.data import DataLoader, Dataset\n",
|
||||
"from transformers import (\n",
|
||||
" AutoTokenizer, AutoModelForCausalLM,\n",
|
||||
" BitsAndBytesConfig, TrainingArguments, Trainer\n",
|
||||
")\n",
|
||||
"from peft import LoraConfig, get_peft_model, TaskType, prepare_model_for_kbit_training\n",
|
||||
"from huggingface_hub import login, HfApi\n",
|
||||
"from google.colab import drive\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
"# === CONFIG ===\n",
|
||||
"HF_TOKEN = \"hf_YOUR_TOKEN_HERE\"\n",
|
||||
"MODEL_ID = 'eulogik/Bharat-Tiny-LLM-v2'\n",
|
||||
"\n",
|
||||
"drive.mount('/content/drive')\n",
|
||||
"DRIVE_DIR = '/content/drive/MyDrive/brahmi_lora'\n",
|
||||
"os.makedirs(DRIVE_DIR, exist_ok=True)\n",
|
||||
"\n",
|
||||
"login(token=HF_TOKEN)\n",
|
||||
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
|
||||
"print(f'Device: {device}')\n",
|
||||
"print(f'Drive: {DRIVE_DIR}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "load_model"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"bnb_config = BitsAndBytesConfig(\n",
|
||||
" load_in_4bit=True,\n",
|
||||
" bnb_4bit_quant_type='nf4',\n",
|
||||
" bnb_4bit_compute_dtype=torch.bfloat16,\n",
|
||||
" bnb_4bit_use_double_quant=True,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"tok = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)\n",
|
||||
"if tok.pad_token is None:\n",
|
||||
" tok.pad_token = tok.eos_token\n",
|
||||
"\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(\n",
|
||||
" MODEL_ID,\n",
|
||||
" quantization_config=bnb_config,\n",
|
||||
" device_map='auto',\n",
|
||||
" token=HF_TOKEN,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"model = prepare_model_for_kbit_training(model)\n",
|
||||
"\n",
|
||||
"new_ids = sorted(\n",
|
||||
" tid for tid, t in tok.added_tokens_decoder.items()\n",
|
||||
" if not str(t).startswith('<') and not getattr(t, 'special', False)\n",
|
||||
")\n",
|
||||
"print(f'Found {len(new_ids)} new tokens')\n",
|
||||
"print(f'Params: {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "lora"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lora_config = LoraConfig(\n",
|
||||
" task_type=TaskType.CAUSAL_LM,\n",
|
||||
" r=16,\n",
|
||||
" lora_alpha=32,\n",
|
||||
" lora_dropout=0.05,\n",
|
||||
" target_modules=['q_proj', 'k_proj', 'v_proj', 'o_proj'],\n",
|
||||
" bias='none',\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"model = get_peft_model(model, lora_config)\n",
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "download_data"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DATA_URL = 'https://huggingface.co/eulogik/Bharat-Tiny-LLM-v2/resolve/main/train_gold_v3.jsonl.gz'\n",
|
||||
"DATA_FILE = '/content/train_gold_v3.jsonl'\n",
|
||||
"\n",
|
||||
"if not os.path.exists(DATA_FILE):\n",
|
||||
" print('Downloading...')\n",
|
||||
" r = requests.get(DATA_URL, stream=True,\n",
|
||||
" headers={'Authorization': f'Bearer {HF_TOKEN}'})\n",
|
||||
" r.raise_for_status()\n",
|
||||
" with open('/content/data.gz', 'wb') as f:\n",
|
||||
" shutil.copyfileobj(r.raw, f)\n",
|
||||
" with gzip.open('/content/data.gz', 'rb') as gz, open(DATA_FILE, 'wb') as f:\n",
|
||||
" shutil.copyfileobj(gz, f)\n",
|
||||
" os.remove('/content/data.gz')\n",
|
||||
"\n",
|
||||
"dataset = load_dataset('text', data_files=DATA_FILE, split='train')\n",
|
||||
"print(f'Loaded {len(dataset)} rows')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "tokenize"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"CHUNKS_FILE = os.path.join(DRIVE_DIR, 'lora_chunks.json')\n",
|
||||
"\n",
|
||||
"if os.path.exists(CHUNKS_FILE):\n",
|
||||
" with open(CHUNKS_FILE) as f:\n",
|
||||
" all_chunks = json.load(f)\n",
|
||||
" print(f'Loaded {len(all_chunks)} chunks from Drive')\n",
|
||||
"else:\n",
|
||||
" all_chunks = []\n",
|
||||
" for i, example in enumerate(dataset):\n",
|
||||
" text = example.get('text', '')\n",
|
||||
" if isinstance(text, str) and text.startswith('{'):\n",
|
||||
" try:\n",
|
||||
" data = json.loads(text)\n",
|
||||
" text = ' '.join(m['content'] for m in data.get('messages', []))\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
" ids = tok.encode(text)\n",
|
||||
" for j in range(0, len(ids), 512):\n",
|
||||
" chunk = ids[j:j+512]\n",
|
||||
" if len(chunk) >= 10:\n",
|
||||
" all_chunks.append(chunk)\n",
|
||||
" if i % 50000 == 0 and i > 0:\n",
|
||||
" print(f' Processed {i}/{len(dataset)} rows...')\n",
|
||||
"\n",
|
||||
" random.shuffle(all_chunks)\n",
|
||||
" if len(all_chunks) > 20000:\n",
|
||||
" all_chunks = all_chunks[:20000]\n",
|
||||
" with open(CHUNKS_FILE, 'w') as f:\n",
|
||||
" json.dump(all_chunks, f)\n",
|
||||
" print(f'Total chunks: {len(all_chunks)} (saved to Drive)')\n",
|
||||
"\n",
|
||||
"split = int(len(all_chunks) * 0.95)\n",
|
||||
"train_chunks = all_chunks[:split]\n",
|
||||
"val_chunks = all_chunks[split:]\n",
|
||||
"print(f'Train: {len(train_chunks)}, Val: {len(val_chunks)}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "dataset"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"class ChunkDataset(Dataset):\n",
|
||||
" def __init__(self, chunks, max_len=512):\n",
|
||||
" self.chunks = chunks\n",
|
||||
" self.max_len = max_len\n",
|
||||
"\n",
|
||||
" def __len__(self):\n",
|
||||
" return len(self.chunks)\n",
|
||||
"\n",
|
||||
" def __getitem__(self, idx):\n",
|
||||
" ids = self.chunks[idx][:self.max_len]\n",
|
||||
" padded = ids + [tok.pad_token_id] * (self.max_len - len(ids))\n",
|
||||
" mask = [1] * len(ids) + [0] * (self.max_len - len(ids))\n",
|
||||
" return {\n",
|
||||
" 'input_ids': torch.tensor(padded, dtype=torch.long),\n",
|
||||
" 'attention_mask': torch.tensor(mask, dtype=torch.long),\n",
|
||||
" 'labels': torch.tensor(padded, dtype=torch.long),\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"train_dataset = ChunkDataset(train_chunks)\n",
|
||||
"val_dataset = ChunkDataset(val_chunks)\n",
|
||||
"print(f'Train: {len(train_dataset)}, Val: {len(val_dataset)}')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "train"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"CKPT_DIR = os.path.join(DRIVE_DIR, 'checkpoints')\n",
|
||||
"os.makedirs(CKPT_DIR, exist_ok=True)\n",
|
||||
"\n",
|
||||
"training_args = TrainingArguments(\n",
|
||||
" output_dir=CKPT_DIR,\n",
|
||||
" num_train_epochs=2,\n",
|
||||
" per_device_train_batch_size=4,\n",
|
||||
" per_device_eval_batch_size=4,\n",
|
||||
" gradient_accumulation_steps=4,\n",
|
||||
" learning_rate=2e-4,\n",
|
||||
" weight_decay=0.01,\n",
|
||||
" warmup_ratio=0.1,\n",
|
||||
" lr_scheduler_type='cosine',\n",
|
||||
" logging_steps=25,\n",
|
||||
" eval_strategy='steps',\n",
|
||||
" eval_steps=100,\n",
|
||||
" save_strategy='steps',\n",
|
||||
" save_steps=100,\n",
|
||||
" save_total_limit=3,\n",
|
||||
" load_best_model_at_end=True,\n",
|
||||
" metric_for_best_model='eval_loss',\n",
|
||||
" greater_is_better=False,\n",
|
||||
" bf16=True,\n",
|
||||
" gradient_checkpointing=True,\n",
|
||||
" gradient_checkpointing_kwargs={'use_reentrant': False},\n",
|
||||
" optim='paged_adamw_8bit',\n",
|
||||
" max_grad_norm=1.0,\n",
|
||||
" report_to='none',\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"trainer = Trainer(\n",
|
||||
" model=model,\n",
|
||||
" args=training_args,\n",
|
||||
" train_dataset=train_dataset,\n",
|
||||
" eval_dataset=val_dataset,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Resume from last checkpoint if exists\n",
|
||||
"checkpoints = [d for d in os.listdir(CKPT_DIR) if d.startswith('checkpoint-')]\n",
|
||||
"resume = None\n",
|
||||
"if checkpoints:\n",
|
||||
" latest = max(checkpoints, key=lambda x: int(x.split('-')[1]))\n",
|
||||
" resume = os.path.join(CKPT_DIR, latest)\n",
|
||||
" print(f'Resuming from {latest}')\n",
|
||||
"\n",
|
||||
"stats = training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps\n",
|
||||
"print(f'Effective batch size: {stats}')\n",
|
||||
"print('Starting training...')\n",
|
||||
"trainer.train(resume_from_checkpoint=resume)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "save_upload"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Save LoRA adapter to Drive and HF\n",
|
||||
"ADAPTER_DIR = os.path.join(DRIVE_DIR, 'lora_adapter')\n",
|
||||
"model.save_pretrained(ADAPTER_DIR)\n",
|
||||
"tok.save_pretrained(ADAPTER_DIR)\n",
|
||||
"print(f'LoRA saved to {ADAPTER_DIR}')\n",
|
||||
"\n",
|
||||
"api = HfApi(token=HF_TOKEN)\n",
|
||||
"api.upload_folder(\n",
|
||||
" folder_path=ADAPTER_DIR,\n",
|
||||
" repo_id='eulogik/Bharat-Tiny-LLM-v2-LoRA',\n",
|
||||
" repo_type='model',\n",
|
||||
" commit_message='LoRA: 2 epochs, rank=16, 2e-4',\n",
|
||||
")\n",
|
||||
"print('Uploaded LoRA to HF!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "merge"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Merge LoRA into base and upload\n",
|
||||
"print('Merging LoRA into base model...')\n",
|
||||
"merged = model.merge_and_unload()\n",
|
||||
"\n",
|
||||
"MERGED_DIR = os.path.join(DRIVE_DIR, 'merged')\n",
|
||||
"merged.save_pretrained(MERGED_DIR)\n",
|
||||
"tok.save_pretrained(MERGED_DIR)\n",
|
||||
"print(f'Merged model saved to {MERGED_DIR}')\n",
|
||||
"\n",
|
||||
"api = HfApi(token=HF_TOKEN)\n",
|
||||
"api.upload_folder(\n",
|
||||
" folder_path=MERGED_DIR,\n",
|
||||
" repo_id='eulogik/Bharat-Tiny-LLM-v2',\n",
|
||||
" repo_type='model',\n",
|
||||
" commit_message='LoRA merged: rank=16, 2 epochs',\n",
|
||||
")\n",
|
||||
"print('Merged model uploaded to HF!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {"id": "generate"},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompts = [\n",
|
||||
" 'मैं आपको बताना चाहता हूँ कि',\n",
|
||||
" 'भारत की राजधानी',\n",
|
||||
" 'नमस्ते, आप कैसे हैं? मैं',\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"for p in prompts:\n",
|
||||
" inputs = tok(p, return_tensors='pt').to(model.device)\n",
|
||||
" out = model.generate(\n",
|
||||
" **inputs,\n",
|
||||
" max_new_tokens=60,\n",
|
||||
" temperature=0.3,\n",
|
||||
" top_p=0.85,\n",
|
||||
" repetition_penalty=1.25,\n",
|
||||
" do_sample=True,\n",
|
||||
" )\n",
|
||||
" gen = tok.decode(out[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)\n",
|
||||
" print(f'Prompt: {p}')\n",
|
||||
" print(f' -> {gen}\\n')"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"accelerator": "GPU",
|
||||
"colab": {"provenance": []},
|
||||
"kernelspec": {"display_name": "Python 3", "name": "python3"},
|
||||
"language_info": {"name": "python"}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
Reference in New Issue
Block a user