129 lines
3.5 KiB
JSON
129 lines
3.5 KiB
JSON
|
{
|
||
|
|
"source_repo": "veyra-ai/Veyra-30M-Base",
|
||
|
|
"legacy_repo": "Jdudeo/Veyra-30M-Base-Legacy",
|
||
|
|
"converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base",
|
||
|
|
"chosen_permute_qk": true,
|
||
|
|
"summary_perm": {
|
||
|
|
"mean_abs_avg": 3.0396581353540697e-06,
|
||
|
|
"max_abs_max": 2.09808349609375e-05,
|
||
|
|
"last_token_mean_abs_avg": 2.9613922184320733e-06,
|
||
|
|
"top1_agree_avg": 1.0
|
||
|
|
},
|
||
|
|
"summary_noperm": {
|
||
|
|
"mean_abs_avg": 0.5820164679655265,
|
||
|
|
"max_abs_max": 8.126092910766602,
|
||
|
|
"last_token_mean_abs_avg": 0.7566529298705973,
|
||
|
|
"top1_agree_avg": 0.757915905543736
|
||
|
|
},
|
||
|
|
"summary_final": {
|
||
|
|
"mean_abs_avg": 3.0396581353540697e-06,
|
||
|
|
"max_abs_max": 2.09808349609375e-05,
|
||
|
|
"last_token_mean_abs_avg": 2.9613922184320733e-06,
|
||
|
|
"top1_agree_avg": 1.0
|
||
|
|
},
|
||
|
|
"rows_final": [
|
||
|
|
{
|
||
|
|
"prompt": "Once upon a time in a digital kingdom,",
|
||
|
|
"tokens": 10,
|
||
|
|
"max_abs": 1.811981201171875e-05,
|
||
|
|
"mean_abs": 2.6239151793561177e-06,
|
||
|
|
"last_token_mean_abs": 3.180772182531655e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "Timmy and Sally went on an adventure to",
|
||
|
|
"tokens": 12,
|
||
|
|
"max_abs": 1.9550323486328125e-05,
|
||
|
|
"mean_abs": 2.7159644560015295e-06,
|
||
|
|
"last_token_mean_abs": 1.937556589837186e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "Question: What is the capital of France?\nAnswer:",
|
||
|
|
"tokens": 14,
|
||
|
|
"max_abs": 2.09808349609375e-05,
|
||
|
|
"mean_abs": 2.5517792892060243e-06,
|
||
|
|
"last_token_mean_abs": 1.6593803593423218e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "I am an AI",
|
||
|
|
"tokens": 4,
|
||
|
|
"max_abs": 1.990795135498047e-05,
|
||
|
|
"mean_abs": 2.7443497856438626e-06,
|
||
|
|
"last_token_mean_abs": 2.7508117454999592e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "1+1=",
|
||
|
|
"tokens": 4,
|
||
|
|
"max_abs": 1.52587890625e-05,
|
||
|
|
"mean_abs": 2.357231778660207e-06,
|
||
|
|
"last_token_mean_abs": 1.859174744822667e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": "def add_numbers(a, b):",
|
||
|
|
"tokens": 11,
|
||
|
|
"max_abs": 2.002716064453125e-05,
|
||
|
|
"mean_abs": 2.8609217679331778e-06,
|
||
|
|
"last_token_mean_abs": 3.9186052163131535e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"prompt": " ",
|
||
|
|
"tokens": 1,
|
||
|
|
"max_abs": 1.8835067749023438e-05,
|
||
|
|
"mean_abs": 5.42344469067757e-06,
|
||
|
|
"last_token_mean_abs": 5.42344469067757e-06,
|
||
|
|
"top1_agree": 1.0
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"llama_config": {
|
||
|
|
"transformers_version": "5.10.2",
|
||
|
|
"architectures": [
|
||
|
|
"LlamaForCausalLM"
|
||
|
|
],
|
||
|
|
"output_hidden_states": false,
|
||
|
|
"return_dict": true,
|
||
|
|
"dtype": "float32",
|
||
|
|
"chunk_size_feed_forward": 0,
|
||
|
|
"is_encoder_decoder": false,
|
||
|
|
"id2label": {
|
||
|
|
"0": "LABEL_0",
|
||
|
|
"1": "LABEL_1"
|
||
|
|
},
|
||
|
|
"label2id": {
|
||
|
|
"LABEL_0": 0,
|
||
|
|
"LABEL_1": 1
|
||
|
|
},
|
||
|
|
"problem_type": null,
|
||
|
|
"vocab_size": 8192,
|
||
|
|
"hidden_size": 512,
|
||
|
|
"intermediate_size": 2048,
|
||
|
|
"num_hidden_layers": 8,
|
||
|
|
"num_attention_heads": 8,
|
||
|
|
"num_key_value_heads": 2,
|
||
|
|
"hidden_act": "silu",
|
||
|
|
"max_position_embeddings": 1024,
|
||
|
|
"initializer_range": 0.02,
|
||
|
|
"rms_norm_eps": 1e-06,
|
||
|
|
"use_cache": true,
|
||
|
|
"pad_token_id": 2,
|
||
|
|
"bos_token_id": 0,
|
||
|
|
"eos_token_id": 1,
|
||
|
|
"pretraining_tp": 1,
|
||
|
|
"tie_word_embeddings": true,
|
||
|
|
"rope_parameters": {
|
||
|
|
"rope_theta": 10000.0,
|
||
|
|
"rope_type": "default"
|
||
|
|
},
|
||
|
|
"attention_bias": false,
|
||
|
|
"attention_dropout": 0.0,
|
||
|
|
"mlp_bias": false,
|
||
|
|
"head_dim": 64,
|
||
|
|
"_name_or_path": "",
|
||
|
|
"model_type": "llama",
|
||
|
|
"output_attentions": false
|
||
|
|
}
|
||
|
|
}
|