{
  "source_repo": "veyra-ai/Veyra-30M-Base-2.5B-Tokens",
  "legacy_repo": "Jdudeo/Veyra-30M-Base-2.5B-Tokens-Legacy",
  "converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base-2.5B-Tokens",
  "chosen_permute_qk": true,
  "summary_perm": {
    "mean_abs_avg": 3.363791880214454e-06,
    "max_abs_max": 2.86102294921875e-05,
    "last_token_mean_abs_avg": 3.3260405416513095e-06,
    "top1_agree_avg": 1.0
  },
  "summary_noperm": {
    "mean_abs_avg": 0.7021041483812626,
    "max_abs_max": 9.765437126159668,
    "last_token_mean_abs_avg": 0.973747450651607,
    "top1_agree_avg": 0.6806431795869555
  },
  "summary_final": {
    "mean_abs_avg": 3.363791880214454e-06,
    "max_abs_max": 2.86102294921875e-05,
    "last_token_mean_abs_avg": 3.3260405416513095e-06,
    "top1_agree_avg": 1.0
  },
  "rows_final": [
    {
      "prompt": "Once upon a time in a digital kingdom,",
      "tokens": 10,
      "max_abs": 2.6226043701171875e-05,
      "mean_abs": 4.017726951133227e-06,
      "last_token_mean_abs": 4.454526788322255e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "Timmy and Sally went on an adventure to",
      "tokens": 12,
      "max_abs": 2.765655517578125e-05,
      "mean_abs": 3.3202672966581304e-06,
      "last_token_mean_abs": 1.8051614461001009e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "Question: What is the capital of France?\nAnswer:",
      "tokens": 14,
      "max_abs": 1.9550323486328125e-05,
      "mean_abs": 3.1002457490103552e-06,
      "last_token_mean_abs": 3.635879693320021e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "I am an AI",
      "tokens": 4,
      "max_abs": 2.86102294921875e-05,
      "mean_abs": 3.60240665031597e-06,
      "last_token_mean_abs": 2.4980258785944898e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "1+1=",
      "tokens": 4,
      "max_abs": 1.9073486328125e-05,
      "mean_abs": 3.561165158316726e-06,
      "last_token_mean_abs": 5.073114607512252e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "def add_numbers(a, b):",
      "tokens": 11,
      "max_abs": 2.7418136596679688e-05,
      "mean_abs": 3.3422006708860863e-06,
      "last_token_mean_abs": 3.2130446925293654e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": " ",
      "tokens": 1,
      "max_abs": 1.633167266845703e-05,
      "mean_abs": 2.6025306851806818e-06,
      "last_token_mean_abs": 2.6025306851806818e-06,
      "top1_agree": 1.0
    }
  ],
  "llama_config": {
    "transformers_version": "5.10.2",
    "architectures": [
      "LlamaForCausalLM"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "vocab_size": 8192,
    "hidden_size": 512,
    "intermediate_size": 2048,
    "num_hidden_layers": 8,
    "num_attention_heads": 8,
    "num_key_value_heads": 2,
    "hidden_act": "silu",
    "max_position_embeddings": 512,
    "initializer_range": 0.02,
    "rms_norm_eps": 1e-06,
    "use_cache": true,
    "pad_token_id": 2,
    "bos_token_id": 0,
    "eos_token_id": 1,
    "pretraining_tp": 1,
    "tie_word_embeddings": true,
    "rope_parameters": {
      "rope_theta": 10000.0,
      "rope_type": "default"
    },
    "attention_bias": false,
    "attention_dropout": 0.0,
    "mlp_bias": false,
    "head_dim": 64,
    "_name_or_path": "",
    "model_type": "llama",
    "output_attentions": false
  }
}