70 lines
920 B
JSON
70 lines
920 B
JSON
{
|
|
"input_config": {
|
|
"model_type": "gpt2",
|
|
"architectures": [
|
|
"GPT2LMHeadModel"
|
|
],
|
|
"activation_function": "gelu",
|
|
"attn_pdrop": 0.1,
|
|
"embd_pdrop": 0.1,
|
|
"resid_pdrop": 0.1,
|
|
"initializer_range": 0.02,
|
|
"layer_norm_epsilon": 1e-05,
|
|
"n_embd": 768,
|
|
"n_inner": 3072,
|
|
"n_head": 12,
|
|
"n_layer": 12,
|
|
"vocab_size": 1,
|
|
"n_ctx": 512,
|
|
"n_positions": 512
|
|
},
|
|
"num_steps_switch": 8320,
|
|
"save_steps": [
|
|
1,
|
|
2,
|
|
4,
|
|
8,
|
|
16,
|
|
32,
|
|
64,
|
|
128,
|
|
256,
|
|
512,
|
|
1000,
|
|
2000,
|
|
3000,
|
|
4000,
|
|
5000,
|
|
6000,
|
|
7000,
|
|
8000,
|
|
8320,
|
|
8321,
|
|
8322,
|
|
8324,
|
|
8328,
|
|
8336,
|
|
8352,
|
|
8384,
|
|
8448,
|
|
8576,
|
|
8832,
|
|
9000,
|
|
10000,
|
|
11000,
|
|
12000,
|
|
13000,
|
|
14000,
|
|
15000,
|
|
16000,
|
|
17000,
|
|
18000,
|
|
19000,
|
|
20000,
|
|
21000,
|
|
22000,
|
|
23000,
|
|
24000,
|
|
24930
|
|
]
|
|
} |