37 lines
752 B
YAML
37 lines
752 B
YAML
|
attention_logit_softcapping: 50.0
|
||
|
|
attention_scores_scalar: 256
|
||
|
|
bias: false
|
||
|
|
block_size: 8192
|
||
|
|
final_logit_softcapping: 30.0
|
||
|
|
gelu_approximate: tanh
|
||
|
|
head_size: 256
|
||
|
|
hf_config:
|
||
|
|
name: gemma-2-2b
|
||
|
|
org: google
|
||
|
|
intermediate_size: 9216
|
||
|
|
lm_head_bias: false
|
||
|
|
mlp_class_name: GemmaMLP
|
||
|
|
n_embd: 2304
|
||
|
|
n_expert: 0
|
||
|
|
n_expert_per_token: 0
|
||
|
|
n_head: 8
|
||
|
|
n_layer: 26
|
||
|
|
n_query_groups: 4
|
||
|
|
name: Gemma-2-2b
|
||
|
|
norm_class_name: RMSNorm
|
||
|
|
norm_eps: 1.0e-05
|
||
|
|
padded_vocab_size: 256000
|
||
|
|
padding_multiple: 512
|
||
|
|
parallel_residual: false
|
||
|
|
post_attention_norm: true
|
||
|
|
post_mlp_norm: true
|
||
|
|
rope_base: 10000
|
||
|
|
rope_condense_ratio: 1
|
||
|
|
rotary_percentage: 1.0
|
||
|
|
scale_embeddings: true
|
||
|
|
shared_attention_norm: false
|
||
|
|
sliding_window_layer_placing: 2
|
||
|
|
sliding_window_size: 4096
|
||
|
|
use_flash_attn: true
|
||
|
|
vocab_size: 256000
|