151 lines
3.3 KiB
JSON
151 lines
3.3 KiB
JSON
{
|
|
"name": "gpt2_strict_small_gpt2init",
|
|
"model_dim": 768,
|
|
"vocab_size": 16384,
|
|
"seq_len": 514,
|
|
"max_positions": 1024,
|
|
"mlp_mult": 4.0,
|
|
"tie_embeddings": true,
|
|
"norm_type": "layernorm",
|
|
"norm_affine": true,
|
|
"norm_eps": 1e-05,
|
|
"positional_embedding": "learned_absolute",
|
|
"embedding_dropout": 0.1,
|
|
"hidden_dropout": 0.1,
|
|
"attn_dropout": 0.1,
|
|
"hf_export_format": "gpt2",
|
|
"blocks": [
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
},
|
|
{
|
|
"type": "plain",
|
|
"heads": 12,
|
|
"attention_mask": "causal",
|
|
"attn_bias": true,
|
|
"ffn_activation": "gelu_new",
|
|
"ffn_pre_norm": true,
|
|
"ffn_bias": true
|
|
}
|
|
],
|
|
"training": {
|
|
"objective": "causal",
|
|
"optimizer": "adamw",
|
|
"lr": 5e-05,
|
|
"weight_decay": 0.0,
|
|
"beta1": 0.9,
|
|
"beta2": 0.999,
|
|
"epsilon": 1e-08,
|
|
"grad_clip": 1.0,
|
|
"steps": 21300,
|
|
"warmup_steps": 213,
|
|
"hold_steps": 0,
|
|
"warmdown_steps": 0,
|
|
"min_lr_fraction": 1e-06,
|
|
"batch_tokens": 8224,
|
|
"compute_dtype": "float32",
|
|
"seed": 42,
|
|
"example_framing": {
|
|
"content_len": 512,
|
|
"bos_id": 1,
|
|
"eos_id": 2
|
|
},
|
|
"weight_init": "gpt2"
|
|
}
|
|
} |