{
  "name": "veyra-30m-base-5b-tokens",
  "created_at": "2026-05-03T17:28:57.455174Z",
  "total_pretraining_tokens": 2500000000,
  "exact_parameter_count": 31988224,
  "parameter_count_millions": 31.99,
  "parameter_breakdown": {
    "token_embedding_tied_lm_head": 4194304,
    "attention_mlp_blocks_4x": 15208448,
    "mlp_only_blocks_4x": 12584960,
    "final_rmsnorm": 512,
    "total": 31988224
  },
  "datasets": [
    {
      "repo": "HuggingFaceTB/cosmopedia-v2",
      "configs_used": [
        "cosmopedia-v2",
        "fineweb-edu-dedup",
        "python-edu"
      ],
      "usage": "1B token Cosmopedia bootstrap plus 1.5B token mixed continuation."
    }
  ],
  "optimizer": {
    "matrix_params": "CosineGatedAdam / CGA-v0",
    "aux_params": "AdamW",
    "cga_peak_lr": 5e-05,
    "cga_max_update_rms": 3.0,
    "aux_peak_lr": 0.0003,
    "lr_schedule": "warmup then cosine decay during continuation"
  },
  "architecture": {
    "model_type": "veyra",
    "architectures": [
      "VeyraForCausalLM"
    ],
    "library_name": "transformers",
    "auto_map": {
      "AutoConfig": "configuration_veyra.VeyraConfig",
      "AutoModelForCausalLM": "modeling_veyra.VeyraForCausalLM"
    },
    "vocab_size": 8192,
    "d_model": 512,
    "hidden_size": 512,
    "n_q_heads": 8,
    "num_attention_heads": 8,
    "n_kv_heads": 2,
    "num_key_value_heads": 2,
    "intermediate_size": 2048,
    "max_seq_len": 1024,
    "max_position_embeddings": 1024,
    "layer_pattern": [
      "A",
      "M",
      "A",
      "M",
      "A",
      "M",
      "A",
      "M"
    ],
    "num_hidden_layers": 8,
    "rms_norm_eps": 1e-06,
    "rope_theta": 10000.0,
    "tie_word_embeddings": true,
    "bos_token_id": 0,
    "eos_token_id": 1,
    "pad_token_id": 2,
    "unk_token_id": 3,
    "dtype": "float32",
    "torch_dtype": "float32",
    "num_parameters": 31988224,
    "parameter_count": 31988224,
    "parameter_count_millions": 31.99,
    "total_params": 31988224,
    "trainable_params": 31988224,
    "training_tokens": 2500000000,
    "notes": "Attention-sparse AMAMAMAM decoder. Early base checkpoint, not instruction tuned."
  },
  "important_warning": "This is an early base model checkpoint, not an instruction-tuned assistant. It can hallucinate, repeat, and fail simple factual/math prompts."
}