{
  "architectures": [
    "LlamaForCausalLM"
  ],
  "attention_bias": false,
  "attention_dropout": 0.0,
  "bos_token_id": 128000,
  "eos_token_id": [
    128001,
    128008,
    128009
  ],
  "head_dim": 128,
  "hidden_act": "silu",
  "hidden_size": 4096,
  "initializer_range": 0.02,
  "intermediate_size": 14336,
  "max_position_embeddings": 131072,
  "mlp_bias": false,
  "model_type": "llama",
  "num_attention_heads": 32,
  "num_hidden_layers": 32,
  "num_key_value_heads": 8,
  "pad_token_id": 128001,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-05,
  "tie_word_embeddings": false,
  "transformers_version": "4.55.0",
  "use_cache": true,
  "vocab_size": 128384,
  "rope_scaling": {
    "factor": 16.0,
    "long_factor": [
      1.0700000524520874,
      1.100000023841858,
      1.1399999856948853,
      1.1399999856948853,
      1.149999976158142,
      1.159999966621399,
      1.159999966621399,
      1.1799999475479126,
      1.1799999475479126,
      1.1899999380111694,
      1.2299998998641968,
      1.2299998998641968,
      1.2999998331069946,
      1.3799998760223389,
      1.429999828338623,
      1.4399996995925903,
      1.869999647140503,
      1.9199997186660767,
      1.9399996995925903,
      2.3799996376037598,
      2.4599997997283936,
      2.499999761581421,
      2.9099998474121094,
      3.2799994945526123,
      3.8399994373321533,
      4.589999198913574,
      4.739999771118164,
      4.800000190734863,
      5.260000228881836,
      5.260000228881836,
      6.310000419616699,
      8.330000877380371,
      9.209999084472656,
      9.889999389648438,
      10.729999542236328,
      11.369999885559082,
      13.589999198913574,
      13.589999198913574,
      15.479998588562012,
      15.479998588562012,
      15.489999771118164,
      15.899998664855957,
      15.989998817443848,
      16.049999237060547,
      16.10999870300293,
      16.10999870300293,
      16.1299991607666,
      16.170000076293945,
      16.190000534057617,
      16.200000762939453,
      16.200000762939453,
      16.30000114440918,
      16.329999923706055,
      16.329999923706055,
      16.34000015258789,
      16.530000686645508,
      16.64000129699707,
      16.68000030517578,
      16.700000762939453,
      16.700002670288086,
      16.790002822875977,
      16.800003051757812,
      16.800003051757812,
      17.01000213623047
    ],
    "original_max_position_embeddings": 8192,
    "rope_type": "longrope",
    "short_factor": [
      1.0700000524520874,
      1.100000023841858,
      1.1399999856948853,
      1.1399999856948853,
      1.149999976158142,
      1.159999966621399,
      1.159999966621399,
      1.1799999475479126,
      1.1799999475479126,
      1.1899999380111694,
      1.2299998998641968,
      1.2299998998641968,
      1.2999998331069946,
      1.3799998760223389,
      1.429999828338623,
      1.4399996995925903,
      1.869999647140503,
      1.9199997186660767,
      1.9399996995925903,
      2.3799996376037598,
      2.4599997997283936,
      2.499999761581421,
      2.9099998474121094,
      3.2799994945526123,
      3.8399994373321533,
      4.589999198913574,
      4.739999771118164,
      4.800000190734863,
      5.260000228881836,
      5.260000228881836,
      6.310000419616699,
      8.330000877380371,
      9.209999084472656,
      9.889999389648438,
      10.729999542236328,
      11.369999885559082,
      13.589999198913574,
      13.589999198913574,
      15.479998588562012,
      15.479998588562012,
      15.489999771118164,
      15.899998664855957,
      15.989998817443848,
      16.049999237060547,
      16.10999870300293,
      16.10999870300293,
      16.1299991607666,
      16.170000076293945,
      16.190000534057617,
      16.200000762939453,
      16.200000762939453,
      16.30000114440918,
      16.329999923706055,
      16.329999923706055,
      16.34000015258789,
      16.530000686645508,
      16.64000129699707,
      16.68000030517578,
      16.700000762939453,
      16.700002670288086,
      16.790002822875977,
      16.800003051757812,
      16.800003051757812,
      17.01000213623047
    ],
    "type": "longrope"
  },
  "rope_theta": 500000,
  "torch_dtype": "bfloat16"
}