{
  "source_repo": "veyra-ai/Veyra-30M-Base",
  "legacy_repo": "Jdudeo/Veyra-30M-Base-Legacy",
  "converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base",
  "chosen_permute_qk": true,
  "summary_perm": {
    "mean_abs_avg": 3.0396581353540697e-06,
    "max_abs_max": 2.09808349609375e-05,
    "last_token_mean_abs_avg": 2.9613922184320733e-06,
    "top1_agree_avg": 1.0
  },
  "summary_noperm": {
    "mean_abs_avg": 0.5820164679655265,
    "max_abs_max": 8.126092910766602,
    "last_token_mean_abs_avg": 0.7566529298705973,
    "top1_agree_avg": 0.757915905543736
  },
  "summary_final": {
    "mean_abs_avg": 3.0396581353540697e-06,
    "max_abs_max": 2.09808349609375e-05,
    "last_token_mean_abs_avg": 2.9613922184320733e-06,
    "top1_agree_avg": 1.0
  },
  "rows_final": [
    {
      "prompt": "Once upon a time in a digital kingdom,",
      "tokens": 10,
      "max_abs": 1.811981201171875e-05,
      "mean_abs": 2.6239151793561177e-06,
      "last_token_mean_abs": 3.180772182531655e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "Timmy and Sally went on an adventure to",
      "tokens": 12,
      "max_abs": 1.9550323486328125e-05,
      "mean_abs": 2.7159644560015295e-06,
      "last_token_mean_abs": 1.937556589837186e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "Question: What is the capital of France?\nAnswer:",
      "tokens": 14,
      "max_abs": 2.09808349609375e-05,
      "mean_abs": 2.5517792892060243e-06,
      "last_token_mean_abs": 1.6593803593423218e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "I am an AI",
      "tokens": 4,
      "max_abs": 1.990795135498047e-05,
      "mean_abs": 2.7443497856438626e-06,
      "last_token_mean_abs": 2.7508117454999592e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "1+1=",
      "tokens": 4,
      "max_abs": 1.52587890625e-05,
      "mean_abs": 2.357231778660207e-06,
      "last_token_mean_abs": 1.859174744822667e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": "def add_numbers(a, b):",
      "tokens": 11,
      "max_abs": 2.002716064453125e-05,
      "mean_abs": 2.8609217679331778e-06,
      "last_token_mean_abs": 3.9186052163131535e-06,
      "top1_agree": 1.0
    },
    {
      "prompt": " ",
      "tokens": 1,
      "max_abs": 1.8835067749023438e-05,
      "mean_abs": 5.42344469067757e-06,
      "last_token_mean_abs": 5.42344469067757e-06,
      "top1_agree": 1.0
    }
  ],
  "llama_config": {
    "transformers_version": "5.10.2",
    "architectures": [
      "LlamaForCausalLM"
    ],
    "output_hidden_states": false,
    "return_dict": true,
    "dtype": "float32",
    "chunk_size_feed_forward": 0,
    "is_encoder_decoder": false,
    "id2label": {
      "0": "LABEL_0",
      "1": "LABEL_1"
    },
    "label2id": {
      "LABEL_0": 0,
      "LABEL_1": 1
    },
    "problem_type": null,
    "vocab_size": 8192,
    "hidden_size": 512,
    "intermediate_size": 2048,
    "num_hidden_layers": 8,
    "num_attention_heads": 8,
    "num_key_value_heads": 2,
    "hidden_act": "silu",
    "max_position_embeddings": 1024,
    "initializer_range": 0.02,
    "rms_norm_eps": 1e-06,
    "use_cache": true,
    "pad_token_id": 2,
    "bos_token_id": 0,
    "eos_token_id": 1,
    "pretraining_tp": 1,
    "tie_word_embeddings": true,
    "rope_parameters": {
      "rope_theta": 10000.0,
      "rope_type": "default"
    },
    "attention_bias": false,
    "attention_dropout": 0.0,
    "mlp_bias": false,
    "head_dim": 64,
    "_name_or_path": "",
    "model_type": "llama",
    "output_attentions": false
  }
}