74 lines
976 B
JSON
74 lines
976 B
JSON
|
{
|
||
|
|
"input_config": {
|
||
|
|
"model_type": "gpt2",
|
||
|
|
"architectures": [
|
||
|
|
"GPT2LMHeadModel"
|
||
|
|
],
|
||
|
|
"activation_function": "gelu",
|
||
|
|
"attn_pdrop": 0.1,
|
||
|
|
"embd_pdrop": 0.1,
|
||
|
|
"resid_pdrop": 0.1,
|
||
|
|
"initializer_range": 0.02,
|
||
|
|
"layer_norm_epsilon": 1e-05,
|
||
|
|
"n_embd": 768,
|
||
|
|
"n_inner": 3072,
|
||
|
|
"n_head": 12,
|
||
|
|
"n_layer": 12,
|
||
|
|
"vocab_size": 1,
|
||
|
|
"n_ctx": 512,
|
||
|
|
"n_positions": 512
|
||
|
|
},
|
||
|
|
"num_steps_switch": 10000,
|
||
|
|
"save_steps": [
|
||
|
|
1,
|
||
|
|
2,
|
||
|
|
4,
|
||
|
|
8,
|
||
|
|
16,
|
||
|
|
32,
|
||
|
|
64,
|
||
|
|
128,
|
||
|
|
256,
|
||
|
|
512,
|
||
|
|
1000,
|
||
|
|
2000,
|
||
|
|
3000,
|
||
|
|
4000,
|
||
|
|
5000,
|
||
|
|
6000,
|
||
|
|
7000,
|
||
|
|
8000,
|
||
|
|
9000,
|
||
|
|
10000,
|
||
|
|
10001,
|
||
|
|
10002,
|
||
|
|
10004,
|
||
|
|
10008,
|
||
|
|
10016,
|
||
|
|
10032,
|
||
|
|
10064,
|
||
|
|
10128,
|
||
|
|
10256,
|
||
|
|
10512,
|
||
|
|
11000,
|
||
|
|
12000,
|
||
|
|
13000,
|
||
|
|
14000,
|
||
|
|
15000,
|
||
|
|
16000,
|
||
|
|
17000,
|
||
|
|
18000,
|
||
|
|
19000,
|
||
|
|
20000,
|
||
|
|
21000,
|
||
|
|
22000,
|
||
|
|
23000,
|
||
|
|
24000,
|
||
|
|
25000,
|
||
|
|
26000,
|
||
|
|
27000,
|
||
|
|
28000,
|
||
|
|
29000,
|
||
|
|
30000
|
||
|
|
]
|
||
|
|
}
|