168 lines
4.1 KiB
JSON
168 lines
4.1 KiB
JSON
{
|
|
"definition": "hard AHA gate zeros / all token x KV-head x layer decisions, token-weighted over prefill and decode",
|
|
"router_granularity": "token",
|
|
"native_router_decisions": 25272464,
|
|
"effective_router_decisions": 202179712,
|
|
"native_router_note": "Native decisions count learned gate logits before broadcast; effective decisions always use token x KV-head x layer for matched sparsity.",
|
|
"sparse_decisions": 15539856,
|
|
"total_decisions": 202179712,
|
|
"sparsity": 0.07686159924889002,
|
|
"full_attention_usage": 0.92313840075111,
|
|
"per_layer": {
|
|
"0": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"1": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"2": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"3": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"4": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"5": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"6": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"7": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"8": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"9": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"10": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"11": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"12": {
|
|
"sparse_decisions": 0,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.0
|
|
},
|
|
"13": {
|
|
"sparse_decisions": 432,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 5.982796137329546e-05
|
|
},
|
|
"14": {
|
|
"sparse_decisions": 3544,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.000490810868303146
|
|
},
|
|
"15": {
|
|
"sparse_decisions": 378032,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.05235389790247599
|
|
},
|
|
"16": {
|
|
"sparse_decisions": 1952,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.00027033375139044616
|
|
},
|
|
"17": {
|
|
"sparse_decisions": 131768,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.01824863614406573
|
|
},
|
|
"18": {
|
|
"sparse_decisions": 33576,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.004649962108957797
|
|
},
|
|
"19": {
|
|
"sparse_decisions": 64112,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.008878912637881292
|
|
},
|
|
"20": {
|
|
"sparse_decisions": 123288,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.017074235420812153
|
|
},
|
|
"21": {
|
|
"sparse_decisions": 287688,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.039842098498982924
|
|
},
|
|
"22": {
|
|
"sparse_decisions": 1334864,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.18486618479306172
|
|
},
|
|
"23": {
|
|
"sparse_decisions": 3498784,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.48454887501274113
|
|
},
|
|
"24": {
|
|
"sparse_decisions": 1545728,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.2140688774944936
|
|
},
|
|
"25": {
|
|
"sparse_decisions": 860456,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.11916511187828777
|
|
},
|
|
"26": {
|
|
"sparse_decisions": 56296,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.007796469707108892
|
|
},
|
|
"27": {
|
|
"sparse_decisions": 7219336,
|
|
"total_decisions": 7220704,
|
|
"sparsity": 0.9998105447889846
|
|
}
|
|
},
|
|
"by_phase": {
|
|
"decode": {
|
|
"sparse_decisions": 270424,
|
|
"total_decisions": 2947840,
|
|
"sparsity": 0.09173632218844985,
|
|
"full_attention_usage": 0.9082636778115502
|
|
},
|
|
"prefill": {
|
|
"sparse_decisions": 15269432,
|
|
"total_decisions": 199231872,
|
|
"sparsity": 0.07664151245840826,
|
|
"full_attention_usage": 0.9233584875415918
|
|
}
|
|
}
|
|
}
|