82 lines
2.3 KiB
JSON
82 lines
2.3 KiB
JSON
|
{
|
||
|
|
"name": "veyra-30m-base-5b-tokens",
|
||
|
|
"created_at": "2026-05-03T17:28:57.455174Z",
|
||
|
|
"total_pretraining_tokens": 2500000000,
|
||
|
|
"exact_parameter_count": 31988224,
|
||
|
|
"parameter_count_millions": 31.99,
|
||
|
|
"parameter_breakdown": {
|
||
|
|
"token_embedding_tied_lm_head": 4194304,
|
||
|
|
"attention_mlp_blocks_4x": 15208448,
|
||
|
|
"mlp_only_blocks_4x": 12584960,
|
||
|
|
"final_rmsnorm": 512,
|
||
|
|
"total": 31988224
|
||
|
|
},
|
||
|
|
"datasets": [
|
||
|
|
{
|
||
|
|
"repo": "HuggingFaceTB/cosmopedia-v2",
|
||
|
|
"configs_used": [
|
||
|
|
"cosmopedia-v2",
|
||
|
|
"fineweb-edu-dedup",
|
||
|
|
"python-edu"
|
||
|
|
],
|
||
|
|
"usage": "1B token Cosmopedia bootstrap plus 1.5B token mixed continuation."
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"optimizer": {
|
||
|
|
"matrix_params": "CosineGatedAdam / CGA-v0",
|
||
|
|
"aux_params": "AdamW",
|
||
|
|
"cga_peak_lr": 5e-05,
|
||
|
|
"cga_max_update_rms": 3.0,
|
||
|
|
"aux_peak_lr": 0.0003,
|
||
|
|
"lr_schedule": "warmup then cosine decay during continuation"
|
||
|
|
},
|
||
|
|
"architecture": {
|
||
|
|
"model_type": "veyra",
|
||
|
|
"architectures": [
|
||
|
|
"VeyraForCausalLM"
|
||
|
|
],
|
||
|
|
"library_name": "transformers",
|
||
|
|
"auto_map": {
|
||
|
|
"AutoConfig": "configuration_veyra.VeyraConfig",
|
||
|
|
"AutoModelForCausalLM": "modeling_veyra.VeyraForCausalLM"
|
||
|
|
},
|
||
|
|
"vocab_size": 8192,
|
||
|
|
"d_model": 512,
|
||
|
|
"hidden_size": 512,
|
||
|
|
"n_q_heads": 8,
|
||
|
|
"num_attention_heads": 8,
|
||
|
|
"n_kv_heads": 2,
|
||
|
|
"num_key_value_heads": 2,
|
||
|
|
"intermediate_size": 2048,
|
||
|
|
"max_seq_len": 1024,
|
||
|
|
"max_position_embeddings": 1024,
|
||
|
|
"layer_pattern": [
|
||
|
|
"A",
|
||
|
|
"M",
|
||
|
|
"A",
|
||
|
|
"M",
|
||
|
|
"A",
|
||
|
|
"M",
|
||
|
|
"A",
|
||
|
|
"M"
|
||
|
|
],
|
||
|
|
"num_hidden_layers": 8,
|
||
|
|
"rms_norm_eps": 1e-06,
|
||
|
|
"rope_theta": 10000.0,
|
||
|
|
"tie_word_embeddings": true,
|
||
|
|
"bos_token_id": 0,
|
||
|
|
"eos_token_id": 1,
|
||
|
|
"pad_token_id": 2,
|
||
|
|
"unk_token_id": 3,
|
||
|
|
"dtype": "float32",
|
||
|
|
"torch_dtype": "float32",
|
||
|
|
"num_parameters": 31988224,
|
||
|
|
"parameter_count": 31988224,
|
||
|
|
"parameter_count_millions": 31.99,
|
||
|
|
"total_params": 31988224,
|
||
|
|
"trainable_params": 31988224,
|
||
|
|
"training_tokens": 2500000000,
|
||
|
|
"notes": "Attention-sparse AMAMAMAM decoder. Early base checkpoint, not instruction tuned."
|
||
|
|
},
|
||
|
|
"important_warning": "This is an early base model checkpoint, not an instruction-tuned assistant. It can hallucinate, repeat, and fail simple factual/math prompts."
|
||
|
|
}
|