129 lines
3.6 KiB
JSON
129 lines
3.6 KiB
JSON
|
{
|
||
|
|
"source_repo": "veyra-ai/Veyra-30M-Base-2.5B-Tokens",
|
||
|
|
"legacy_repo": "Jdudeo/Veyra-30M-Base-2.5B-Tokens-Legacy",
|
||
|
|
"converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base-2.5B-Tokens",
|
||
|
|
"chosen_permute_qk": true,
|
||
|
|
"summary_perm": {
|
||
|
|
"mean_abs_avg": 3.363791880214454e-06,
|
||
|
|
"max_abs_max": 2.86102294921875e-05,
|
||
|
|
"last_token_mean_abs_avg": 3.3260405416513095e-06,
|
||
|
|
"top1_agree_avg": 1.0
|
||
|
|
},
|
||
|
|
"summary_noperm": {
|
||
|
|
"mean_abs_avg": 0.7021041483812626,
|
||
|
|
"max_abs_max": 9.765437126159668,
|
||
|
|
"last_token_mean_abs_avg": 0.973747450651607,
|
||
|
|
"top1_agree_avg": 0.6806431795869555
|
||
|
|
},
|
||
|
|
"summary_final": {
|
||
|
|
"mean_abs_avg": 3.363791880214454e-06,
|
||
|
|
"max_abs_max": 2.86102294921875e-05,
|
||
|
|
"last_token_mean_abs_avg": 3.3260405416513095e-06,
|
||
|
|
"top1_agree_avg": 1.0
|
||
|
|
},
|
||
|
|
"rows_final": [
|
||
|
|
{
|
||
|
|
"prompt": "Once upon a time in a digital kingdom,",
|
||
|
|
"tokens": 10,
|
||
|
|
"max_abs": 2.6226043701171875e-05,
|
||
|
|
"mean_abs": 4.017726951133227e-06,
|
||
|
|
"last_token_mean_abs": 4.454526788322255e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "Timmy and Sally went on an adventure to",
|
||
|
|
"tokens": 12,
|
||
|
|
"max_abs": 2.765655517578125e-05,
|
||
|
|
"mean_abs": 3.3202672966581304e-06,
|
||
|
|
"last_token_mean_abs": 1.8051614461001009e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "Question: What is the capital of France?\nAnswer:",
|
||
|
|
"tokens": 14,
|
||
|
|
"max_abs": 1.9550323486328125e-05,
|
||
|
|
"mean_abs": 3.1002457490103552e-06,
|
||
|
|
"last_token_mean_abs": 3.635879693320021e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "I am an AI",
|
||
|
|
"tokens": 4,
|
||
|
|
"max_abs": 2.86102294921875e-05,
|
||
|
|
"mean_abs": 3.60240665031597e-06,
|
||
|
|
"last_token_mean_abs": 2.4980258785944898e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "1+1=",
|
||
|
|
"tokens": 4,
|
||
|
|
"max_abs": 1.9073486328125e-05,
|
||
|
|
"mean_abs": 3.561165158316726e-06,
|
||
|
|
"last_token_mean_abs": 5.073114607512252e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "def add_numbers(a, b):",
|
||
|
|
"tokens": 11,
|
||
|
|
"max_abs": 2.7418136596679688e-05,
|
||
|
|
"mean_abs": 3.3422006708860863e-06,
|
||
|
|
"last_token_mean_abs": 3.2130446925293654e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": " ",
|
||
|
|
"tokens": 1,
|
||
|
|
"max_abs": 1.633167266845703e-05,
|
||
|
|
"mean_abs": 2.6025306851806818e-06,
|
||
|
|
"last_token_mean_abs": 2.6025306851806818e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"llama_config": {
|
||
|
|
"transformers_version": "5.10.2",
|
||
|
|
"architectures": [
|
||
|
|
"LlamaForCausalLM"
|
||
|
|
],
|
||
|
|
"output_hidden_states": false,
|
||
|
|
"return_dict": true,
|
||
|
|
"dtype": "float32",
|
||
|
|
"chunk_size_feed_forward": 0,
|
||
|
|
"is_encoder_decoder": false,
|
||
|
|
"id2label": {
|
||
|
|
"0": "LABEL_0",
|
||
|
|
"1": "LABEL_1"
|
||
|
|
},
|
||
|
|
"label2id": {
|
||
|
|
"LABEL_0": 0,
|
||
|
|
"LABEL_1": 1
|
||
|
|
},
|
||
|
|
"problem_type": null,
|
||
|
|
"vocab_size": 8192,
|
||
|
|
"hidden_size": 512,
|
||
|
|
"intermediate_size": 2048,
|
||
|
|
"num_hidden_layers": 8,
|
||
|
|
"num_attention_heads": 8,
|
||
|
|
"num_key_value_heads": 2,
|
||
|
|
"hidden_act": "silu",
|
||
|
|
"max_position_embeddings": 512,
|
||
|
|
"initializer_range": 0.02,
|
||
|
|
"rms_norm_eps": 1e-06,
|
||
|
|
"use_cache": true,
|
||
|
|
"pad_token_id": 2,
|
||
|
|
"bos_token_id": 0,
|
||
|
|
"eos_token_id": 1,
|
||
|
|
"pretraining_tp": 1,
|
||
|
|
"tie_word_embeddings": true,
|
||
|
|
"rope_parameters": {
|
||
|
|
"rope_theta": 10000.0,
|
||
|
|
"rope_type": "default"
|
||
|
|
},
|
||
|
|
"attention_bias": false,
|
||
|
|
"attention_dropout": 0.0,
|
||
|
|
"mlp_bias": false,
|
||
|
|
"head_dim": 64,
|
||
|
|
"_name_or_path": "",
|
||
|
|
"model_type": "llama",
|
||
|
|
"output_attentions": false
|
||
|
|
}
|
||
|
|
}
|