142 lines
3.4 KiB
JSON
142 lines
3.4 KiB
JSON
|
{
|
||
|
|
"task": "text-to-SQL (schema + question -> SQLite query)",
|
||
|
|
"dataset": "b-mc2/sql-create-context",
|
||
|
|
"base_model": "Qwen/Qwen2.5-1.5B-Instruct",
|
||
|
|
"method": "QLoRA (4-bit NF4 frozen base + LoRA adapters)",
|
||
|
|
"primary_metric": "normalized exact match",
|
||
|
|
"splits": {
|
||
|
|
"dataset": "b-mc2/sql-create-context",
|
||
|
|
"seed": 42,
|
||
|
|
"raw_rows": 78577,
|
||
|
|
"unique_rows": 78573,
|
||
|
|
"n_train": 12000,
|
||
|
|
"n_val": 750,
|
||
|
|
"n_test": 1000,
|
||
|
|
"train_test_overlap": 0,
|
||
|
|
"val_test_overlap": 0,
|
||
|
|
"train_val_overlap": 0,
|
||
|
|
"unique_keys_per_split": [
|
||
|
|
12000,
|
||
|
|
750,
|
||
|
|
1000
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"training": {
|
||
|
|
"epochs": 2,
|
||
|
|
"lr": 0.0002,
|
||
|
|
"lora_r": 16,
|
||
|
|
"lora_alpha": 32,
|
||
|
|
"effective_batch": 32,
|
||
|
|
"n_train": 12000,
|
||
|
|
"n_val": 750,
|
||
|
|
"max_len": 512,
|
||
|
|
"train_minutes": 11.7,
|
||
|
|
"peak_vram_gb": 30.96,
|
||
|
|
"base_model": "Qwen/Qwen2.5-1.5B-Instruct"
|
||
|
|
},
|
||
|
|
"baseline": {
|
||
|
|
"exact_match": 0.497,
|
||
|
|
"token_f1": 0.9245807106744601,
|
||
|
|
"format_compliance": 0.267,
|
||
|
|
"n": 1000,
|
||
|
|
"tag": "baseline_0shot",
|
||
|
|
"quant": "4bit",
|
||
|
|
"fewshot": 0,
|
||
|
|
"adapter": null,
|
||
|
|
"split": "test",
|
||
|
|
"wall_seconds": 46.4,
|
||
|
|
"peak_vram_gb": 2.12
|
||
|
|
},
|
||
|
|
"baseline_3shot": {
|
||
|
|
"exact_match": 0.523,
|
||
|
|
"token_f1": 0.9212539742018538,
|
||
|
|
"format_compliance": 0.993,
|
||
|
|
"n": 1000,
|
||
|
|
"tag": "baseline_3shot",
|
||
|
|
"quant": "4bit",
|
||
|
|
"fewshot": 3,
|
||
|
|
"adapter": null,
|
||
|
|
"split": "test",
|
||
|
|
"wall_seconds": 56.4,
|
||
|
|
"peak_vram_gb": 3.08
|
||
|
|
},
|
||
|
|
"baseline_fp16": {
|
||
|
|
"exact_match": 0.573,
|
||
|
|
"token_f1": 0.9382126511712477,
|
||
|
|
"format_compliance": 0.994,
|
||
|
|
"n": 1000,
|
||
|
|
"tag": "baseline_fp16_0shot",
|
||
|
|
"quant": "fp16",
|
||
|
|
"fewshot": 0,
|
||
|
|
"adapter": null,
|
||
|
|
"split": "test",
|
||
|
|
"wall_seconds": 47.4,
|
||
|
|
"peak_vram_gb": 4.05
|
||
|
|
},
|
||
|
|
"finetuned": {
|
||
|
|
"exact_match": 0.748,
|
||
|
|
"token_f1": 0.973163722916554,
|
||
|
|
"format_compliance": 0.999,
|
||
|
|
"n": 1000,
|
||
|
|
"tag": "finetuned",
|
||
|
|
"quant": "4bit",
|
||
|
|
"fewshot": 0,
|
||
|
|
"adapter": "outputs/adapter",
|
||
|
|
"split": "test",
|
||
|
|
"wall_seconds": 68.0,
|
||
|
|
"peak_vram_gb": 2.79
|
||
|
|
},
|
||
|
|
"delta": {
|
||
|
|
"exact_match_abs": 0.251,
|
||
|
|
"exact_match_rel": 0.505,
|
||
|
|
"token_f1_abs": 0.0486,
|
||
|
|
"format_compliance_abs": 0.732,
|
||
|
|
"improved": true
|
||
|
|
},
|
||
|
|
"quantization": [
|
||
|
|
{
|
||
|
|
"precision": "fp16",
|
||
|
|
"weights_vram_gb": 3.09,
|
||
|
|
"peak_vram_gb": 3.3,
|
||
|
|
"load_seconds": 2.2,
|
||
|
|
"latency_ms_per_request_bs1": 1862.4,
|
||
|
|
"decode_tokens_per_sec_bs1": 34.4,
|
||
|
|
"batch16_tokens_per_sec": 436.5,
|
||
|
|
"gen_tokens": 64,
|
||
|
|
"exact_match": 0.7667,
|
||
|
|
"token_f1": 0.9737,
|
||
|
|
"quality_n": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"precision": "8bit",
|
||
|
|
"weights_vram_gb": 1.8,
|
||
|
|
"peak_vram_gb": 2.1,
|
||
|
|
"load_seconds": 20.3,
|
||
|
|
"latency_ms_per_request_bs1": 27963.2,
|
||
|
|
"decode_tokens_per_sec_bs1": 2.3,
|
||
|
|
"batch16_tokens_per_sec": 16.7,
|
||
|
|
"gen_tokens": 64,
|
||
|
|
"exact_match": 0.7533,
|
||
|
|
"token_f1": 0.9726,
|
||
|
|
"quality_n": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"precision": "4bit",
|
||
|
|
"weights_vram_gb": 1.16,
|
||
|
|
"peak_vram_gb": 1.54,
|
||
|
|
"load_seconds": 5.5,
|
||
|
|
"latency_ms_per_request_bs1": 2320.7,
|
||
|
|
"decode_tokens_per_sec_bs1": 27.6,
|
||
|
|
"batch16_tokens_per_sec": 369.2,
|
||
|
|
"gen_tokens": 64,
|
||
|
|
"exact_match": 0.7533,
|
||
|
|
"token_f1": 0.9732,
|
||
|
|
"quality_n": 300
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"hub": {
|
||
|
|
"adapter": "blaze-star/qwen2.5-1.5b-sql-qlora",
|
||
|
|
"merged": "blaze-star/qwen2.5-1.5b-sql-qlora-merged"
|
||
|
|
},
|
||
|
|
"code": "https://github.com/harshb20/qwen2.5-1.5b-sql-qlora"
|
||
|
|
}
|