1337d25c94
Model: xd2010/Qwen1.5-MOE-aux-free-sft-math7k-1e-3-gamma-part2-test Source: Original Platform
1667 lines
46 KiB
JSON
1667 lines
46 KiB
JSON
{
|
|
"metadata": {
|
|
"total_moe_layers": 24,
|
|
"save_timestamp": "2026-03-23T17:53:46.497250",
|
|
"model_type": "Qwen2MoeForCausalLM",
|
|
"pytorch_version": "2.6.0+cu124",
|
|
"description": "Auxiliary-loss-free MoE bias states saved during training"
|
|
},
|
|
"moe_bias_states": {
|
|
"model.layers.0.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.7796038389205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7815569639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.1.mlp": {
|
|
"bias_values": [
|
|
0.8362444639205933,
|
|
-0.8596819639205933,
|
|
-0.8245257139205933,
|
|
0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.5017744898796082,
|
|
-0.8362444639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7522600889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.2.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8323382139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8245257139205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.7913225889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.3.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8323382139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8206194639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933,
|
|
0.8440569639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.6917132139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.7756975889205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.4.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8089007139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8206194639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8323382139205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.5.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
-0.8362444639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8440569639205933,
|
|
-0.8596819639205933,
|
|
0.8479632139205933,
|
|
0.8518694639205933,
|
|
-0.8440569639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.6.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8167132139205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.6702290177345276,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8440569639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.7.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8323382139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8284319639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.6555807590484619,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8284319639205933,
|
|
0.8323382139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.8.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8440569639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8479632139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8518694639205933,
|
|
0.8030413389205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8167132139205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8440569639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.9.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8089007139205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7190569639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8323382139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7717913389205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.10.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
0.5996362566947937,
|
|
-0.8596819639205933,
|
|
-0.8323382139205933,
|
|
-0.7952288389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8245257139205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8206194639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7913225889205933,
|
|
0.7366350889205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8030413389205933,
|
|
0.8596819639205933,
|
|
0.8128069639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.11.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8089007139205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8440569639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7991350889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7366350889205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.12.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7913225889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8206194639205933,
|
|
-0.8518694639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8089007139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7717913389205933,
|
|
0.8596819639205933,
|
|
0.8401507139205933,
|
|
0.8596819639205933,
|
|
-0.7991350889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.13.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
0.6799007654190063,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7678850889205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7913225889205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8167132139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.14.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7991350889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.7600725889205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.6096830368041992,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.7288225889205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8245257139205933,
|
|
0.8479632139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.15.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7190569639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8401507139205933,
|
|
-0.8401507139205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8167132139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8401507139205933,
|
|
0.8596819639205933,
|
|
-0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8206194639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.16.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
0.8401507139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.6155422925949097,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7913225889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7561663389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8167132139205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8284319639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.17.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8245257139205933,
|
|
0.8596819639205933,
|
|
-0.8440569639205933,
|
|
0.8596819639205933,
|
|
-0.8479632139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8323382139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.7483538389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.6318623423576355,
|
|
0.8167132139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8089007139205933,
|
|
-0.8596819639205933,
|
|
-0.7952288389205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7639788389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7952288389205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.18.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8165258169174194,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8421038389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.7737444639205933,
|
|
0.8596819639205933,
|
|
-0.8399633169174194,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8089007139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.7874163389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.7835100889205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.19.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7522600889205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8362444639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.7913225889205933,
|
|
-0.8596819639205933,
|
|
0.8440569639205933,
|
|
-0.8596819639205933,
|
|
0.6203702688217163,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8440569639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.20.mlp": {
|
|
"bias_values": [
|
|
-0.8557757139205933,
|
|
0.8518694639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7952288389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.7874163389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7835100889205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7756975889205933,
|
|
-0.8596819639205933,
|
|
-0.8499163389205933,
|
|
0.8206194639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8128069639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.21.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7151507139205933,
|
|
-0.8596819639205933,
|
|
-0.7600725889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7835100889205933,
|
|
-0.8440569639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8284319639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7756975889205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.7678850889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.22.mlp": {
|
|
"bias_values": [
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.7053850889205933,
|
|
-0.8596819639205933,
|
|
-0.8323382139205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.7913225889205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.7385882139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
0.8401507139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8125258684158325,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
},
|
|
"model.layers.23.mlp": {
|
|
"bias_values": [
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.7639788389205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.7561663389205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
0.8596819639205933,
|
|
0.8557757139205933,
|
|
-0.8596819639205933,
|
|
-0.388006329536438,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8323382139205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8479632139205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8518694639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8401507139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8089007139205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933,
|
|
0.8596819639205933,
|
|
-0.8596819639205933,
|
|
-0.8596819639205933
|
|
],
|
|
"bias_update_speed": 0.001,
|
|
"num_experts": 60,
|
|
"module_type": "AuxFreeQwen2MoeSparseMoeBlock",
|
|
"device": "cuda:0",
|
|
"dtype": "torch.float32"
|
|
}
|
|
}
|
|
} |