67 lines
887 B
JSON
67 lines
887 B
JSON
|
{
|
||
|
|
"input_config": {
|
||
|
|
"model_type": "gpt2",
|
||
|
|
"architectures": [
|
||
|
|
"GPT2LMHeadModel"
|
||
|
|
],
|
||
|
|
"activation_function": "gelu",
|
||
|
|
"attn_pdrop": 0.1,
|
||
|
|
"embd_pdrop": 0.1,
|
||
|
|
"resid_pdrop": 0.1,
|
||
|
|
"initializer_range": 0.02,
|
||
|
|
"layer_norm_epsilon": 1e-05,
|
||
|
|
"n_embd": 768,
|
||
|
|
"n_inner": 3072,
|
||
|
|
"n_head": 12,
|
||
|
|
"n_layer": 12,
|
||
|
|
"vocab_size": 1,
|
||
|
|
"n_ctx": 512,
|
||
|
|
"n_positions": 512
|
||
|
|
},
|
||
|
|
"num_steps_switch": 7220,
|
||
|
|
"save_steps": [
|
||
|
|
1,
|
||
|
|
2,
|
||
|
|
4,
|
||
|
|
8,
|
||
|
|
16,
|
||
|
|
32,
|
||
|
|
64,
|
||
|
|
128,
|
||
|
|
256,
|
||
|
|
512,
|
||
|
|
1000,
|
||
|
|
2000,
|
||
|
|
3000,
|
||
|
|
4000,
|
||
|
|
5000,
|
||
|
|
6000,
|
||
|
|
7000,
|
||
|
|
7220,
|
||
|
|
7221,
|
||
|
|
7222,
|
||
|
|
7224,
|
||
|
|
7228,
|
||
|
|
7236,
|
||
|
|
7252,
|
||
|
|
7284,
|
||
|
|
7348,
|
||
|
|
7476,
|
||
|
|
7732,
|
||
|
|
8000,
|
||
|
|
9000,
|
||
|
|
10000,
|
||
|
|
11000,
|
||
|
|
12000,
|
||
|
|
13000,
|
||
|
|
14000,
|
||
|
|
15000,
|
||
|
|
16000,
|
||
|
|
17000,
|
||
|
|
18000,
|
||
|
|
19000,
|
||
|
|
20000,
|
||
|
|
21000,
|
||
|
|
21640
|
||
|
|
]
|
||
|
|
}
|