67 lines
887 B
JSON
67 lines
887 B
JSON
{
|
|
"input_config": {
|
|
"model_type": "gpt2",
|
|
"architectures": [
|
|
"GPT2LMHeadModel"
|
|
],
|
|
"activation_function": "gelu",
|
|
"attn_pdrop": 0.1,
|
|
"embd_pdrop": 0.1,
|
|
"resid_pdrop": 0.1,
|
|
"initializer_range": 0.02,
|
|
"layer_norm_epsilon": 1e-05,
|
|
"n_embd": 768,
|
|
"n_inner": 3072,
|
|
"n_head": 12,
|
|
"n_layer": 12,
|
|
"vocab_size": 1,
|
|
"n_ctx": 512,
|
|
"n_positions": 512
|
|
},
|
|
"num_steps_switch": 7260,
|
|
"save_steps": [
|
|
1,
|
|
2,
|
|
4,
|
|
8,
|
|
16,
|
|
32,
|
|
64,
|
|
128,
|
|
256,
|
|
512,
|
|
1000,
|
|
2000,
|
|
3000,
|
|
4000,
|
|
5000,
|
|
6000,
|
|
7000,
|
|
7260,
|
|
7261,
|
|
7262,
|
|
7264,
|
|
7268,
|
|
7276,
|
|
7292,
|
|
7324,
|
|
7388,
|
|
7516,
|
|
7772,
|
|
8000,
|
|
9000,
|
|
10000,
|
|
11000,
|
|
12000,
|
|
13000,
|
|
14000,
|
|
15000,
|
|
16000,
|
|
17000,
|
|
18000,
|
|
19000,
|
|
20000,
|
|
21000,
|
|
21750
|
|
]
|
|
} |