19 lines
407 B
JSON
19 lines
407 B
JSON
{
|
|
"source_ckpt": "checkpoints/500m_chat_final.pt",
|
|
"source_iter": 405,
|
|
"native_cfg": {
|
|
"vocab_size": 32000,
|
|
"dim": 1280,
|
|
"n_layers": 26,
|
|
"n_heads": 20,
|
|
"n_kv_heads": 4,
|
|
"max_seq_len": 2048,
|
|
"ffn_multiple_of": 256,
|
|
"ffn_dim_multiplier": null,
|
|
"rope_theta": 10000.0,
|
|
"dropout": 0.0,
|
|
"norm_eps": 1e-05,
|
|
"grad_checkpoint": false
|
|
},
|
|
"dtype": "bfloat16"
|
|
} |