初始化项目,由ModelHub XC社区提供模型
Model: KookiesXy/Neo50M Source: Original Platform
This commit is contained in:
21
training_configs/model_neo50m.yaml
Normal file
21
training_configs/model_neo50m.yaml
Normal file
@@ -0,0 +1,21 @@
|
||||
model:
|
||||
name: Neo50M
|
||||
architecture: llama
|
||||
tokenizer_id: TinyLlama/TinyLlama-1.1B-Chat-v1.0
|
||||
hidden_size: 512
|
||||
intermediate_size: 1536
|
||||
num_hidden_layers: 12
|
||||
num_attention_heads: 8
|
||||
num_key_value_heads: 2
|
||||
max_position_embeddings: 16384
|
||||
rope_theta: 500000.0
|
||||
rms_norm_eps: 0.000001
|
||||
initializer_range: 0.02
|
||||
tie_word_embeddings: true
|
||||
use_cache: false
|
||||
attention_dropout: 0.0
|
||||
attention_bias: false
|
||||
mlp_bias: false
|
||||
attn_implementation: sdpa
|
||||
gradient_checkpointing: false
|
||||
torch_dtype: bfloat16
|
||||
Reference in New Issue
Block a user