22 lines
519 B
YAML
22 lines
519 B
YAML
|
model:
|
||
|
|
name: Neo50M
|
||
|
|
architecture: llama
|
||
|
|
tokenizer_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
|
||
|
|
hidden_size: 512
|
||
|
|
intermediate_size: 1536
|
||
|
|
num_hidden_layers: 12
|
||
|
|
num_attention_heads: 8
|
||
|
|
num_key_value_heads: 2
|
||
|
|
max_position_embeddings: 16384
|
||
|
|
rope_theta: 500000.0
|
||
|
|
rms_norm_eps: 0.000001
|
||
|
|
initializer_range: 0.02
|
||
|
|
tie_word_embeddings: true
|
||
|
|
use_cache: false
|
||
|
|
attention_dropout: 0.0
|
||
|
|
attention_bias: false
|
||
|
|
mlp_bias: false
|
||
|
|
attn_implementation: sdpa
|
||
|
|
gradient_checkpointing: false
|
||
|
|
torch_dtype: bfloat16
|