{
  "task": "text-to-SQL (schema + question -> SQLite query)",
  "dataset": "b-mc2/sql-create-context",
  "base_model": "Qwen/Qwen2.5-1.5B-Instruct",
  "method": "QLoRA (4-bit NF4 frozen base + LoRA adapters)",
  "primary_metric": "normalized exact match",
  "splits": {
    "dataset": "b-mc2/sql-create-context",
    "seed": 42,
    "raw_rows": 78577,
    "unique_rows": 78573,
    "n_train": 12000,
    "n_val": 750,
    "n_test": 1000,
    "train_test_overlap": 0,
    "val_test_overlap": 0,
    "train_val_overlap": 0,
    "unique_keys_per_split": [
      12000,
      750,
      1000
    ]
  },
  "training": {
    "epochs": 2,
    "lr": 0.0002,
    "lora_r": 16,
    "lora_alpha": 32,
    "effective_batch": 32,
    "n_train": 12000,
    "n_val": 750,
    "max_len": 512,
    "train_minutes": 11.7,
    "peak_vram_gb": 30.96,
    "base_model": "Qwen/Qwen2.5-1.5B-Instruct"
  },
  "baseline": {
    "exact_match": 0.497,
    "token_f1": 0.9245807106744601,
    "format_compliance": 0.267,
    "n": 1000,
    "tag": "baseline_0shot",
    "quant": "4bit",
    "fewshot": 0,
    "adapter": null,
    "split": "test",
    "wall_seconds": 46.4,
    "peak_vram_gb": 2.12
  },
  "baseline_3shot": {
    "exact_match": 0.523,
    "token_f1": 0.9212539742018538,
    "format_compliance": 0.993,
    "n": 1000,
    "tag": "baseline_3shot",
    "quant": "4bit",
    "fewshot": 3,
    "adapter": null,
    "split": "test",
    "wall_seconds": 56.4,
    "peak_vram_gb": 3.08
  },
  "baseline_fp16": {
    "exact_match": 0.573,
    "token_f1": 0.9382126511712477,
    "format_compliance": 0.994,
    "n": 1000,
    "tag": "baseline_fp16_0shot",
    "quant": "fp16",
    "fewshot": 0,
    "adapter": null,
    "split": "test",
    "wall_seconds": 47.4,
    "peak_vram_gb": 4.05
  },
  "finetuned": {
    "exact_match": 0.748,
    "token_f1": 0.973163722916554,
    "format_compliance": 0.999,
    "n": 1000,
    "tag": "finetuned",
    "quant": "4bit",
    "fewshot": 0,
    "adapter": "outputs/adapter",
    "split": "test",
    "wall_seconds": 68.0,
    "peak_vram_gb": 2.79
  },
  "delta": {
    "exact_match_abs": 0.251,
    "exact_match_rel": 0.505,
    "token_f1_abs": 0.0486,
    "format_compliance_abs": 0.732,
    "improved": true
  },
  "quantization": [
    {
      "precision": "fp16",
      "weights_vram_gb": 3.09,
      "peak_vram_gb": 3.3,
      "load_seconds": 2.2,
      "latency_ms_per_request_bs1": 1862.4,
      "decode_tokens_per_sec_bs1": 34.4,
      "batch16_tokens_per_sec": 436.5,
      "gen_tokens": 64,
      "exact_match": 0.7667,
      "token_f1": 0.9737,
      "quality_n": 300
    },
    {
      "precision": "8bit",
      "weights_vram_gb": 1.8,
      "peak_vram_gb": 2.1,
      "load_seconds": 20.3,
      "latency_ms_per_request_bs1": 27963.2,
      "decode_tokens_per_sec_bs1": 2.3,
      "batch16_tokens_per_sec": 16.7,
      "gen_tokens": 64,
      "exact_match": 0.7533,
      "token_f1": 0.9726,
      "quality_n": 300
    },
    {
      "precision": "4bit",
      "weights_vram_gb": 1.16,
      "peak_vram_gb": 1.54,
      "load_seconds": 5.5,
      "latency_ms_per_request_bs1": 2320.7,
      "decode_tokens_per_sec_bs1": 27.6,
      "batch16_tokens_per_sec": 369.2,
      "gen_tokens": 64,
      "exact_match": 0.7533,
      "token_f1": 0.9732,
      "quality_n": 300
    }
  ],
  "hub": {
    "adapter": "blaze-star/qwen2.5-1.5b-sql-qlora",
    "merged": "blaze-star/qwen2.5-1.5b-sql-qlora-merged"
  },
  "code": "https://github.com/harshb20/qwen2.5-1.5b-sql-qlora"
}