model:
  name: Neo50M
  architecture: llama
  tokenizer_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
  hidden_size: 512
  intermediate_size: 1536
  num_hidden_layers: 12
  num_attention_heads: 8
  num_key_value_heads: 2
  max_position_embeddings: 16384
  rope_theta: 500000.0
  rms_norm_eps: 0.000001
  initializer_range: 0.02
  tie_word_embeddings: true
  use_cache: false
  attention_dropout: 0.0
  attention_bias: false
  mlp_bias: false
  attn_implementation: sdpa
  gradient_checkpointing: false
  torch_dtype: bfloat16