{
  "source_ckpt": "checkpoints/500m_sft_final.pt",
  "source_iter": 1192,
  "native_cfg": {
    "vocab_size": 32000,
    "dim": 1280,
    "n_layers": 26,
    "n_heads": 20,
    "n_kv_heads": 4,
    "max_seq_len": 2048,
    "ffn_multiple_of": 256,
    "ffn_dim_multiplier": null,
    "rope_theta": 10000.0,
    "dropout": 0.0,
    "norm_eps": 1e-05,
    "grad_checkpoint": false
  },
  "dtype": "bfloat16"
}