17 lines
500 B
JSON
17 lines
500 B
JSON
|
{
|
||
|
|
"model_name": "veyra2-15m-base-1b-tokens",
|
||
|
|
"source_folder": "/content/drive/MyDrive/veyra_runs/veyra2_15m_base_pretrain_1b/checkpoints/final_hf",
|
||
|
|
"params": 14685888,
|
||
|
|
"train_tokens": 2000158720,
|
||
|
|
"quick_eval_loss": 2.8223165422677994,
|
||
|
|
"quick_eval_ppl": 16.815760006000545,
|
||
|
|
"vocab_size": 8192,
|
||
|
|
"context_length": 1024,
|
||
|
|
"architecture": {
|
||
|
|
"hidden_size": 448,
|
||
|
|
"intermediate_size": 1024,
|
||
|
|
"num_hidden_layers": 6,
|
||
|
|
"num_attention_heads": 7,
|
||
|
|
"num_key_value_heads": 1
|
||
|
|
}
|
||
|
|
}
|