{
  "metadata": {
    "total_moe_layers": 24,
    "save_timestamp": "2026-03-23T17:53:46.497250",
    "model_type": "Qwen2MoeForCausalLM",
    "pytorch_version": "2.6.0+cu124",
    "description": "Auxiliary-loss-free MoE bias states saved during training"
  },
  "moe_bias_states": {
    "model.layers.0.mlp": {
      "bias_values": [
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8362444639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.7796038389205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7815569639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.1.mlp": {
      "bias_values": [
        0.8362444639205933,
        -0.8596819639205933,
        -0.8245257139205933,
        0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.5017744898796082,
        -0.8362444639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7522600889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.2.mlp": {
      "bias_values": [
        0.8596819639205933,
        0.8401507139205933,
        -0.8596819639205933,
        -0.8323382139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8245257139205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.7913225889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.3.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8323382139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8206194639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933,
        0.8440569639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.6917132139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.7756975889205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.4.mlp": {
      "bias_values": [
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8089007139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8596819639205933,
        -0.8362444639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8206194639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8323382139205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.5.mlp": {
      "bias_values": [
        0.8596819639205933,
        -0.8362444639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8440569639205933,
        -0.8596819639205933,
        0.8479632139205933,
        0.8518694639205933,
        -0.8440569639205933,
        0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.6.mlp": {
      "bias_values": [
        0.8596819639205933,
        0.8596819639205933,
        -0.8167132139205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.6702290177345276,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8440569639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.7.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8323382139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8284319639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.6555807590484619,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8284319639205933,
        0.8323382139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.8.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8440569639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8479632139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.8518694639205933,
        0.8030413389205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8167132139205933,
        0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8440569639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.9.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8089007139205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7190569639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8323382139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7717913389205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.10.mlp": {
      "bias_values": [
        -0.8596819639205933,
        0.5996362566947937,
        -0.8596819639205933,
        -0.8323382139205933,
        -0.7952288389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8245257139205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8206194639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7913225889205933,
        0.7366350889205933,
        -0.8401507139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8557757139205933,
        0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8030413389205933,
        0.8596819639205933,
        0.8128069639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.11.mlp": {
      "bias_values": [
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8089007139205933,
        0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8440569639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7991350889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7366350889205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.12.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7913225889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8206194639205933,
        -0.8518694639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8089007139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8362444639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7717913389205933,
        0.8596819639205933,
        0.8401507139205933,
        0.8596819639205933,
        -0.7991350889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.13.mlp": {
      "bias_values": [
        -0.8596819639205933,
        0.6799007654190063,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.7678850889205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8362444639205933,
        0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7913225889205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8167132139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.14.mlp": {
      "bias_values": [
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.7991350889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.7600725889205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.6096830368041992,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.7288225889205933,
        0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8245257139205933,
        0.8479632139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.15.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.7190569639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8401507139205933,
        -0.8401507139205933,
        -0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8167132139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        -0.8401507139205933,
        0.8596819639205933,
        -0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8596819639205933,
        -0.8206194639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.16.mlp": {
      "bias_values": [
        -0.8596819639205933,
        0.8401507139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.6155422925949097,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7913225889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7561663389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8167132139205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8284319639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.17.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8245257139205933,
        0.8596819639205933,
        -0.8440569639205933,
        0.8596819639205933,
        -0.8479632139205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8323382139205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.7483538389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.6318623423576355,
        0.8167132139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8089007139205933,
        -0.8596819639205933,
        -0.7952288389205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7639788389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.7952288389205933,
        0.8596819639205933,
        0.8557757139205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.18.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8165258169174194,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8421038389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.7737444639205933,
        0.8596819639205933,
        -0.8399633169174194,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8518694639205933,
        -0.8089007139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.7874163389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.7835100889205933,
        0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.19.mlp": {
      "bias_values": [
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7522600889205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8362444639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.7913225889205933,
        -0.8596819639205933,
        0.8440569639205933,
        -0.8596819639205933,
        0.6203702688217163,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8518694639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8440569639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8518694639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.20.mlp": {
      "bias_values": [
        -0.8557757139205933,
        0.8518694639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7952288389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        -0.8557757139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.7874163389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7835100889205933,
        -0.8518694639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7756975889205933,
        -0.8596819639205933,
        -0.8499163389205933,
        0.8206194639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8128069639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.21.mlp": {
      "bias_values": [
        -0.8596819639205933,
        0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7151507139205933,
        -0.8596819639205933,
        -0.7600725889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7835100889205933,
        -0.8440569639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8284319639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8557757139205933,
        -0.8401507139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.7756975889205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.7678850889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.22.mlp": {
      "bias_values": [
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8401507139205933,
        -0.7053850889205933,
        -0.8596819639205933,
        -0.8323382139205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.7913225889205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.7385882139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        0.8401507139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8557757139205933,
        -0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8125258684158325,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    },
    "model.layers.23.mlp": {
      "bias_values": [
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.7639788389205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.7561663389205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        0.8596819639205933,
        0.8557757139205933,
        -0.8596819639205933,
        -0.388006329536438,
        -0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8323382139205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8479632139205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8518694639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8401507139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        0.8089007139205933,
        -0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933,
        0.8596819639205933,
        -0.8596819639205933,
        -0.8596819639205933
      ],
      "bias_update_speed": 0.001,
      "num_experts": 60,
      "module_type": "AuxFreeQwen2MoeSparseMoeBlock",
      "device": "cuda:0",
      "dtype": "torch.float32"
    }
  }
}