62439 lines
1.7 MiB
62439 lines
1.7 MiB
{
|
|
"measurement": {
|
|
"model.layers.0.self_attn": [
|
|
{
|
|
"accuracy": 0.893648700886651,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.907640319905783,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9196362611102431,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9459652645807517,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.946334001461142,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9476359276787231,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9549954668863824,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.956701953944407,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9664377413297954,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9679571554849022,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9719989747789345,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9773675229792532,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9739424167691093,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9797968387505726,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9839276195454755,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.989946322221505,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9854142696988818,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941176721875212,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961664159904773,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.0.mlp": [
|
|
{
|
|
"accuracy": 0.8949737423344662,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9023757961235548,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.919411802958501,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9250818855668369,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.95734549843167,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9608159108381522,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9693435313866327,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9771821260158169,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.979731474267809,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9786837054416537,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98138285648862,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9891378644755796,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9907404418642584,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993926382525579,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99446065140594,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9963663903579704,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981245036531043,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.1.self_attn": [
|
|
{
|
|
"accuracy": 0.8987936644177688,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9075590885783497,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9200696317773116,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.94466037518884,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9456692906586748,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9476314385078455,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9551406736044508,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9575842858145112,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9660790268527835,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9673481175774022,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9733174656958956,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9778770412269392,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.974967837627781,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9799472785701877,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9844503263011575,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890619423730593,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9862945915542936,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950041872890372,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965011606017422,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.1.mlp": [
|
|
{
|
|
"accuracy": 0.9571290168126947,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9612742379601849,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9644467866930523,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.964791260071491,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9914404379547035,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9936220313324348,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9945022363068634,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951376328612432,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961974459369421,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965270539427078,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9969821132805297,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99783900426088,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981071585666781,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982829373054157,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981865582246284,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982787630533015,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985438172435878,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.2.self_attn": [
|
|
{
|
|
"accuracy": 0.9927235057724542,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9928812040873852,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941201964577072,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9949724506893146,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959106653083214,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959403647624544,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9971368907227818,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972557004412489,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9974387123182072,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9974856387125328,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997663105850255,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9977892724196672,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982802076877928,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998476602547606,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.999081017276407,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9991998228230697,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9991919901053494,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996777575985394,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9997872566658005,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.2.mlp": [
|
|
{
|
|
"accuracy": 0.9866840624201455,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.987057142204752,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892221709198662,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899010710222157,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993340472531456,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993861836503799,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948078712730325,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965365829473165,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9968621040553491,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9966118606021873,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9970331034028755,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982752163854648,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985228424132067,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990684121271203,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9991206869959378,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9993120947972209,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9997530742854762,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.3.self_attn": [
|
|
{
|
|
"accuracy": 0.990764206630717,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9912796362033603,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923754579412114,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941113181044593,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947355947714593,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948662300410337,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995808427062768,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959196003190683,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9963423732766196,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964640566736067,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972703480590625,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975565299973823,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975307930604016,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9978076685097461,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985672855630248,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989001643292508,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9987220556297536,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995550521117892,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996730677896841,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.3.mlp": [
|
|
{
|
|
"accuracy": 0.9823102801793108,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9828021370707766,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9854761164047217,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9863110618714831,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910502494312823,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917626336069876,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.992919434266361,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9953528948887986,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9957823679892739,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954428697299016,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960108646007843,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976790182776504,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980135790875034,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9987465504979666,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988173698442743,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990465397483939,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996694107949703,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.4.self_attn": [
|
|
{
|
|
"accuracy": 0.9896868751139233,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902426185650065,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9914267798518076,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929275298858747,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9940039805430723,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944513246637622,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9952298220871997,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956828307323647,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960232374356374,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961321806584141,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9970844067375813,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9973314616731123,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9974143113358878,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976581675674472,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99852704507985,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988273102008909,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998687424989552,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995228175585477,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996583332915754,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.4.mlp": [
|
|
{
|
|
"accuracy": 0.9782556757379911,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9789032122904533,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9823563833181795,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98341018870767,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890307452970821,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899247727773496,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913822962338791,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.994266726480993,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947995110353651,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944101855906314,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995119627338442,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9971529651635425,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975681974871182,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984455614160795,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985486383160797,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988446216157172,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995867812509747,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.5.self_attn": [
|
|
{
|
|
"accuracy": 0.9864892647846749,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9870703478931988,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9886728218560549,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9912153213824096,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9928887421142703,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9931625498238167,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948053060981789,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951194139185214,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995685532664586,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959068029297908,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965109790270952,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9968041769032808,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9969521340558698,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972412716229692,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99825070564341,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9986158731687618,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985545483497461,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9993651939408952,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996419745546422,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.5.mlp": [
|
|
{
|
|
"accuracy": 0.9741528061169543,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.974932648094469,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9790864163568538,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9803391301906422,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9870105549742124,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9880504037596678,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9897935249916229,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9932253561350272,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9938369115054804,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9933986412973976,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9942237760773615,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9966456971413112,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9971262766293397,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981728592936537,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982944574960704,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9986445978682145,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995115070839682,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.6.self_attn": [
|
|
{
|
|
"accuracy": 0.9849805038814482,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9857031190522799,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9875402976513693,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9905975606831673,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923567865534049,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923406477502891,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944012549852854,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.994444224346233,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9949445316298423,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9953512568884578,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962265982329356,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965019909187073,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9966510403594983,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9969324421652249,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981921479798315,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984882973030357,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985811512604797,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.999250332754424,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9996303597129698,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.6.mlp": [
|
|
{
|
|
"accuracy": 0.9715909987786099,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9724220427145299,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9771649930135984,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9785952651873231,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9857135999359583,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9868831679862189,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888509437686911,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9925233546742483,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9932158342685158,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9927295576168322,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9936509703678128,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9963017185708802,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9968365344562029,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9979805491870808,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981205844577696,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985275985152264,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9994606216917562,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.7.self_attn": [
|
|
{
|
|
"accuracy": 0.9817928779429119,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9826188418327978,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9848534433955425,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.988524671276345,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909426557316788,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910408153419236,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9936385371379162,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993736001062452,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9943761398501106,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948335300983959,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955105170129651,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959104370791465,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961537933712336,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964660923915768,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9978839689038547,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981967715368802,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984113365828403,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989892480125969,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.999565374948648,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.7.mlp": [
|
|
{
|
|
"accuracy": 0.9683402055188229,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.969293144531548,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9744191953030071,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9759467045139325,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9841223999759868,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9854270072261754,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9875291685181621,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917217636554453,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9924724908962258,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919096430101874,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929408406789758,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9958785920599965,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964826465216711,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9977500131231194,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9978997589113485,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9983327312848383,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.999402847758335,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.8.self_attn": [
|
|
{
|
|
"accuracy": 0.9812522998177692,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9820844099954947,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9839425302649799,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9882299120684988,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9900897194043194,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.990344744347232,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929596173675045,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993291257120865,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9938924511355397,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.994253650589503,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950106285487939,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954475205342629,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955202457495034,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960316607459007,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975682604800615,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980175398277903,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981318534670496,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998976778363086,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995210681364896,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.8.mlp": [
|
|
{
|
|
"accuracy": 0.9660537193872427,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.967053475299556,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9725423862079257,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9741759401206908,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9829769125208259,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9843717229454533,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9866237784245688,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911370483031007,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919386924930701,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913359457675955,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.992433363522746,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955869920331201,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962290810592669,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975906277663613,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9977549816555294,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99821921383141,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9993512322314353,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.9.self_attn": [
|
|
{
|
|
"accuracy": 0.9788888592371031,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.979685742702139,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9814297161800297,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9856838251856205,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.989116406680918,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9893769212557297,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9924206560918767,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.992760036875935,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9934621135165033,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9939353014465029,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947472068138028,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951050378823358,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9953085821930712,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956341984706294,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997459819871246,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9978819965433917,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998156496394107,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998847417447618,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9995237096514565,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.9.mlp": [
|
|
{
|
|
"accuracy": 0.9637203136164891,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9648244897589872,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9706952222868016,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9724594056116123,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9817620026633928,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9832546998207506,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9857082760304605,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9904883745812664,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913337958829576,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9907063720747828,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9918807046733013,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9952663337983387,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959555486045581,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9974121707675391,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997586914466841,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980802573937628,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9992975088320427,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.10.self_attn": [
|
|
{
|
|
"accuracy": 0.9757592889333242,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9767382253745669,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9788646645549881,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9840242695553523,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9879006163070077,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9882532914512252,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9921008524068288,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9926131115000891,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929971037260992,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9932944712866294,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993977920201264,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9943498858912406,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947341374538251,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951328340501181,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9970582743408158,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975686568681053,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9978928915163698,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9987045606555368,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9994329656458975,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.10.mlp": [
|
|
{
|
|
"accuracy": 0.9621513052598426,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.963333536135523,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9697476796021587,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9716803818254879,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9809128715607681,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9825276646174883,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9851904507040193,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899500107000533,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909074719946244,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902680985335457,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9915220866862097,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950415729912684,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9957727208905118,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972715327197588,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9974768413537133,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980351940642944,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9992708913893071,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.11.self_attn": [
|
|
{
|
|
"accuracy": 0.9720823717744727,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9733222414690413,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.975832394422277,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.981664082819694,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.985370115229958,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9859467925945002,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9894892006475282,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.990257480549381,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911924630991722,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.991585842885175,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929489056016073,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9934006264359739,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9937730974542272,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9943289988012495,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964620773496694,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997112926438843,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972665775628564,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982880388821573,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9992598743395463,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.11.mlp": [
|
|
{
|
|
"accuracy": 0.9607131737039277,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9619707324002919,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9685673406837803,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9705858152163657,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9801831524819136,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9818474902330261,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9846200701339465,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9895316675599468,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.990526917978729,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.989895188421207,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.991188030052734,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948526626624363,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956064626550007,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99715705369109,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9973855374050081,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9979701325432152,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9992322900078235,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.12.self_attn": [
|
|
{
|
|
"accuracy": 0.9684477893047427,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9698352818132231,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9732198911768041,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9789277682767102,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9837384673913843,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9841225381303382,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9884589728654215,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9889393041988737,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9898009717121328,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902483127324989,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917622994337427,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9922265035688485,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9925971793280425,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9931405129058188,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9957913435309341,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99644620194541,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9966932975221425,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981692837992389,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990703587613289,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.12.mlp": [
|
|
{
|
|
"accuracy": 0.9566547479480505,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9580882887699103,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9650715496391058,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9671825780288169,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9781345885345026,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9799985146934265,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9829207373488891,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9884878138668443,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9895687559864631,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888370130701285,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902801895759216,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9942982324987257,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951402363367379,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9968428737238834,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9970928177813461,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976898267509808,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9991357476239117,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.13.self_attn": [
|
|
{
|
|
"accuracy": 0.9684169794383802,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9696670207440069,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9724430151185707,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9794576871944102,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9838014003379565,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9842974005855227,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9895115582958648,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902006392367184,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909024215058276,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917205331209851,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9920856905120768,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9925373429064884,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9930801329717628,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9935873312091357,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961396617275712,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9967780449292868,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972789736685196,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982054253157816,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9992875282738456,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.13.mlp": [
|
|
{
|
|
"accuracy": 0.9529289841455848,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9544367339265973,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9621385216320816,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9645361087628102,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9761344929176726,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9781865440309048,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9814245895923752,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.987363262910788,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9885729813801223,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9878127577126419,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9894002435581857,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993783649767896,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947088850098417,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965564195850962,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9968356265085995,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9975159614971888,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990697039879466,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.14.self_attn": [
|
|
{
|
|
"accuracy": 0.9641322645505792,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9658698517044908,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9689987783173197,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9764699113408202,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9807388833969047,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9817256947587195,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869102909180679,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9881771871280906,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9893347276619783,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9898594750679637,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9907180134354061,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913013153954556,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919415679025022,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9925027607670543,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950792989325955,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962417516416233,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961632958821658,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9979278004642478,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989671096708136,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.14.mlp": [
|
|
{
|
|
"accuracy": 0.949354276159092,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9509656417526697,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9591436736089618,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9616885332292632,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9743993059290867,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9766254142220867,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9800560679286718,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9863806701216259,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876607114106024,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869228449503058,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9886158166551277,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9933115444268639,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9942862065508962,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962511662589876,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9965815110144353,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9972983938580575,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988980756515,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.15.self_attn": [
|
|
{
|
|
"accuracy": 0.9607511900953556,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9621540077618862,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9660266462321344,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9728861059503335,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9788440923255525,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9793920574317637,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98499083793477,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9857311479159092,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9872394960716759,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.987753185786699,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9893208128449164,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9901923450249198,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9907579334256681,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913785900724562,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9945269333027107,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9957010119442681,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955731555574426,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976417796171614,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988614656398759,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.15.mlp": [
|
|
{
|
|
"accuracy": 0.9450141473820335,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9466919732329092,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9555978385829612,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9583613345105397,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9722626028876555,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9745566143507236,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.97828436662492,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9853475005610993,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9866934682634708,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9858602356949919,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876487214167259,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9927935867178205,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9938389849849045,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960123123723621,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.996340551879257,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9971121865704557,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989225209359766,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.16.self_attn": [
|
|
{
|
|
"accuracy": 0.9612767202289481,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9624278198339438,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9666875200836282,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9739447416443574,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.980184206788085,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9808493492830741,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9870004695595095,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9878931655548513,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9887068702741281,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888761922913162,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902446198669311,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909048316215998,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9915182177026413,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923085471566179,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995083556461491,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960624276121196,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962824908648863,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997802622015833,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990399017176395,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.16.mlp": [
|
|
{
|
|
"accuracy": 0.938417885060373,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9403196202689096,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9502678997814655,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.953463710158279,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9689377064963705,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9715393658139204,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9757145258941149,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98346198384503,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9849971436748379,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9841628493250985,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9861856615661007,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919238965468187,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9931090511871796,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954793177128426,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959113273500023,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9967796598732668,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9987705998906964,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.17.self_attn": [
|
|
{
|
|
"accuracy": 0.9586278024668756,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9604273262972894,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9644740538573578,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.973357011417025,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9782964021065518,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9790577416945445,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9849650717193359,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9860310838802865,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869455792941153,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876664129674042,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892881202609524,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899113913998008,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9904662106235168,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911841043320141,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9943343285887846,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995567274304401,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955653401557356,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997660272417737,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988467652675401,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.17.mlp": [
|
|
{
|
|
"accuracy": 0.9316650299649489,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9338378837626231,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9448322049881283,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9485885143083961,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9654491223198802,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9683350305023947,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9730381782313711,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9814552948565075,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9832163978564111,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9823712778620814,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98461890205937,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910016162075886,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923203375533616,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9949448966715289,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954338128016772,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964314942419725,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9986026544724346,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.18.self_attn": [
|
|
{
|
|
"accuracy": 0.9569948224448844,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9586803686658019,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9628864861044445,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9700939966071593,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9773965421083727,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9784271336209617,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9834641970596031,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9847993677187907,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9862319972905281,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869650247212696,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9885926066015503,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.989399680304096,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9896327188532603,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.990581254989497,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9932304782498824,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951676490569585,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941313269450084,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976250872775716,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9983332144972672,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.18.mlp": [
|
|
{
|
|
"accuracy": 0.9254861513250753,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9278848045750668,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.939896970791252,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.944188978523016,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9623849112540483,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9655444276960272,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9706776110749495,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9796028717077876,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9815827435568759,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9808089839000451,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9832627560061059,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902116334506947,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9916419286378905,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944359899841642,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950487006970338,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9961626258629718,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984365756635701,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.19.self_attn": [
|
|
{
|
|
"accuracy": 0.95781120736348,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.959114997010482,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9633006224114644,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9713525100562134,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9776565027178118,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9785075935192014,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9842612063512206,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9853657964420947,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9866936848449864,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9875479785382355,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890527046334586,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899703866164935,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9905281453638485,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9914180126874462,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944312093093207,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955346610911778,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956750110110366,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9973838936084097,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988699828789226,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.19.mlp": [
|
|
{
|
|
"accuracy": 0.9222120334835429,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9246534909072676,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9370473137027339,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9414813018550998,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9606579941158232,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9639779263617176,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9693550907663608,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9787138911258233,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9807556194596385,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9798755072743485,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9825038989809783,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9897075766256374,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9912505476049295,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941645303348962,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947755085549465,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959219854668175,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9983584174065312,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.20.self_attn": [
|
|
{
|
|
"accuracy": 0.9606317537591645,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9621753562241793,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9658298302245767,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9732479803068074,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9792457794476497,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9805042028034988,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9851283590358338,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9867370901629329,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9878318707801794,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9882330988827897,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9899942806039593,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9907368548753622,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910602873975509,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.991812973724384,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9945894426568166,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959098866062337,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956173081514671,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9977941521823308,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988178378736944,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.20.mlp": [
|
|
{
|
|
"accuracy": 0.9208069339786705,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.923151061330971,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9356271050086147,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9397877997865802,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.960099447714655,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9633125477519474,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9686747722719845,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9788245344044346,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9807572595677093,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9796901778072903,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9822352145259318,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9896630236486855,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911442811993,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.994264080017609,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947587737666541,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995868146713627,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984258940676227,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.21.self_attn": [
|
|
{
|
|
"accuracy": 0.9646951217988604,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.966109805887467,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9695159268418425,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9760359940854343,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9819418337980383,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9824662318355158,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869473035398283,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876788753624025,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888254870464536,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.98898859258349,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9904326248276782,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9913903545882357,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9915054561815372,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923164283268546,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9950486005185858,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.996065060657106,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9958613950416053,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980173610444916,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988622526043879,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.21.mlp": [
|
|
{
|
|
"accuracy": 0.9198688354931379,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9221995157238684,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9345135722113282,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9384046706714129,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9596971583209539,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9628805608341569,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.968111219570825,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9788478590351971,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9807202974824529,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9795030151052695,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9820441562976492,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9895880563478721,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910581371697941,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9942730876224998,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947100772007712,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995750414722256,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984387571702859,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.22.self_attn": [
|
|
{
|
|
"accuracy": 0.9700527208808222,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9713233700512272,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.974679631622214,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9794778611118856,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9831417335295364,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9841211775415822,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9867493111248079,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9880293208468509,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890092376031374,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9893504335839105,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917076805753535,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9922008875659422,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9924224708474388,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9930184699938094,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9952682150699395,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9963412584344807,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9958134077636427,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982653830359739,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998882547468192,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.22.mlp": [
|
|
{
|
|
"accuracy": 0.9192072094271058,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9214393441614351,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9334900255658125,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9372236575735242,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9591717811203316,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9623905922237196,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9675098172339954,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9786467255142174,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9805289936300955,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9792461133513012,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9817899022937605,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9894624673761427,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909419414195183,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9942407321527993,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9946480505903693,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9956688466972035,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984434554475898,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.23.self_attn": [
|
|
{
|
|
"accuracy": 0.967110109858607,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9685465163110119,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9719149626203274,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9771949089573402,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9819119751061264,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.983264238034424,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.985921165660808,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876456008733887,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9886357925685221,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892037849334118,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9916052495521542,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9921106818671289,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9925297090636664,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993053306825459,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954197926226219,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964758525112349,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9960902930041285,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982409670238236,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989609639527962,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.23.mlp": [
|
|
{
|
|
"accuracy": 0.9174738410664232,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.919812290684173,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9317592700061045,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.935453051799222,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9582668631092498,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9615686420155198,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9666586202618322,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9781673251602211,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9801108978296581,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9787792159048351,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.981389647839885,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892142510629798,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.990736430633421,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941003706649338,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9945088422391564,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955069033159433,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984050017818319,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.24.self_attn": [
|
|
{
|
|
"accuracy": 0.9670127036147996,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9682941274031213,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9715245832738123,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9768591511406397,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9819275129978594,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9831022745194403,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9864346011982936,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9878436869843618,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9885056996472964,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.989142831463955,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911194481200686,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919792210291091,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9921301834677395,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993141142063235,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9955529490069143,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9966187642176488,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962676784856931,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.998163651883308,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9990210516047419,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.24.mlp": [
|
|
{
|
|
"accuracy": 0.9161045586592272,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9185225106775761,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9303537785614792,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9340069119475389,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.957608575001359,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9609911775118426,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9660250326912654,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9778528932089868,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9797793648352748,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9784304076119473,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9811139989056086,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890379725073121,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9905997387525675,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9939981482212285,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.994429152201567,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954070558673457,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9983710186114829,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.25.self_attn": [
|
|
{
|
|
"accuracy": 0.964759472659544,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9661269402621608,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9701274526550582,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9753424673292198,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9818012450674647,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9830015311507803,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9863354913204124,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9879132596481788,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892629399944685,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9895438870758211,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9914572980794075,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9918991863531502,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9924871840171123,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9929787813146648,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9953872297743434,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964583219690738,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.996110736073828,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981813731137663,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989384419413431,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.25.mlp": [
|
|
{
|
|
"accuracy": 0.9141801315310755,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9166429578081557,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9283984083878367,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9321081477560496,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9566785423575264,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9600312250029099,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9650454630977229,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9772528163402489,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.979248303184776,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9779869014103162,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9806568835322794,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888322100190348,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9903772422602695,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993877491019176,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9943400900810957,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.99532736009477,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9983556924424576,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.26.self_attn": [
|
|
{
|
|
"accuracy": 0.966179788896912,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9675700778239652,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9708114050720867,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9768774325616265,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9820629412700471,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9827608322528633,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9870124382800177,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9879549262358954,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9889288402505612,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892898481339216,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9911948870622406,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9917074302701574,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9922178539349452,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9927546359157484,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951838131895975,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964085837801624,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9959435005378174,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980942702937969,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9989540236174913,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.26.mlp": [
|
|
{
|
|
"accuracy": 0.9119538653053736,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9144667696795965,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9262061458277075,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9299508195958639,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9553948665331853,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9589224018548664,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9639908445900992,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9765911116509846,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9786329281663424,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9773249296275409,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9801120952163872,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9884856860888632,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9900993786899275,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9936728263764005,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941634642033789,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9951560393613028,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982651505300677,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.27.self_attn": [
|
|
{
|
|
"accuracy": 0.9648983853035852,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9663207070215752,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9701925369077607,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9762639110221675,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.981060051800389,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9820240110060886,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9860233350313807,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9873275530867671,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9887377839829576,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892669800472887,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9908756973516,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9918085846461748,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919320217970955,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9926946583007904,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9953376881557664,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9964522919004881,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9962301632193359,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9981360389042253,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.999000269423337,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.27.mlp": [
|
|
{
|
|
"accuracy": 0.9096943216496393,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9123458781916844,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9242802074081019,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9281321897318489,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9541935897187183,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9578169033323464,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9630049984706075,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9758465096452519,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9779806089048323,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9767026228242015,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9795504914792744,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9881696234898347,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9898247486960731,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9934678037748917,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9939966508138337,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995016257719774,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9982026951436541,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.28.self_attn": [
|
|
{
|
|
"accuracy": 0.9588921317144444,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9606568942728796,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9652924616085856,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9722532321159777,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9789059131259197,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9796413719458016,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9847067561686823,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9856548179921351,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9869243582024386,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9875306883140614,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9897275494754707,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9902258172974383,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9909687454714194,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9916422974906469,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9947387195241294,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9957732843896864,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.995717662129257,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9977273816782001,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9988222851687553,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.28.mlp": [
|
|
{
|
|
"accuracy": 0.9026325282297636,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9055269007620058,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9189013934841281,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9232583138111391,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.950616097763965,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9545402133739308,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9603373365182626,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9737823557500777,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.97611838706622,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9748444991480363,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9779487504182678,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.987215901970079,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9890194768086076,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9928743334634131,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9935300047264287,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9946858318105928,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980486457321891,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.29.self_attn": [
|
|
{
|
|
"accuracy": 0.9466490504380903,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9486936562155422,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9558809939771891,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9648195690426388,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9726870815808836,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9732231219348154,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9809688415476366,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9816111375234629,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9836317750772363,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.984283749553326,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9862153427185196,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9872727194604906,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9879910280731948,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9892708948617311,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9934665916340524,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9945631078812048,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9948307117569799,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9971918211453349,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9985990498251723,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.29.mlp": [
|
|
{
|
|
"accuracy": 0.901279168301507,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9042282790729874,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9179903813882878,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9224478480847258,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9497262947261333,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9537225632291091,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9597617504432013,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9731450167141462,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9755005167895242,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9742648528006516,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9773369537372338,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9868389577359745,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9886687370507341,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9926819352335051,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9933077193502533,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9944885691113182,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9980146456754914,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.30.self_attn": [
|
|
{
|
|
"accuracy": 0.9406793137129984,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9429967025000798,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9547522818963778,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9629674215654009,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9706006906926632,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9720876527656066,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9769263143504137,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9788594822350302,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9781166090580978,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9792905510158131,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9857550591818596,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.987330460822896,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9876325348098027,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9888965698253167,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9930148357280383,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941437575402424,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9941515274541942,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997075852116962,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9984381716134713,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.30.mlp": [
|
|
{
|
|
"accuracy": 0.8897434343632898,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.8927277910866236,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9078390186554507,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9125531260904512,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9436142025025267,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9484581319909346,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9550479244636862,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9694714521695125,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9724417713127638,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9709315553895737,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9745572299549454,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9850480102100655,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9870389812674961,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9916096411547378,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9923844088212048,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9938080814225894,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9976935863225279,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.31.self_attn": [
|
|
{
|
|
"accuracy": 0.9416899386989444,
|
|
"total_bits": 89665536,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9441070644871185,
|
|
"total_bits": 92221440,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9520620680169055,
|
|
"total_bits": 95758848,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.966899961331173,
|
|
"total_bits": 112272384,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.971686082745069,
|
|
"total_bits": 132913152,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9724544484756494,
|
|
"total_bits": 132980224,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64,
|
|
"3": 64
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.981647342884619,
|
|
"total_bits": 169613312,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.982805434467369,
|
|
"total_bits": 169745920,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9847021714637154,
|
|
"total_bits": 171195392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9854566185784183,
|
|
"total_bits": 173563904,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9864706255397514,
|
|
"total_bits": 174923264,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9873422583271014,
|
|
"total_bits": 175750144,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9886315461052092,
|
|
"total_bits": 179253248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 64
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 64,
|
|
"4": 64
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9894115809155138,
|
|
"total_bits": 181592064,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9937483897236618,
|
|
"total_bits": 220469248,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9946627592864005,
|
|
"total_bits": 223535104,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9954790754464308,
|
|
"total_bits": 253499392,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9970372415907485,
|
|
"total_bits": 265838592,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 32
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"6": 32
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9987065431257514,
|
|
"total_bits": 337385472,
|
|
"q_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"k_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"v_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"o_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.layers.31.mlp": [
|
|
{
|
|
"accuracy": 0.8837233181846769,
|
|
"total_bits": 395461696,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.8868422480790239,
|
|
"total_bits": 409224256,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
6,
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.2,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9018708073387021,
|
|
"total_bits": 457272320,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9065089186555461,
|
|
"total_bits": 512977920,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"3": 64,
|
|
"2": 64
|
|
},
|
|
"bits": [
|
|
3,
|
|
2
|
|
],
|
|
"bits_prop": [
|
|
0.3,
|
|
0.7
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9406428999806705,
|
|
"total_bits": 578397280,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128,
|
|
"3": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9460245866916681,
|
|
"total_bits": 592872448,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9525223821401596,
|
|
"total_bits": 637454432,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32,
|
|
"3": 32
|
|
},
|
|
"bits": [
|
|
4,
|
|
3
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.96840099323737,
|
|
"total_bits": 728741472,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9715074199791017,
|
|
"total_bits": 739476480,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9694849297796425,
|
|
"total_bits": 751543392,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 128,
|
|
"4": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9735955698905807,
|
|
"total_bits": 766018560,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"5": 32,
|
|
"4": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
5,
|
|
4
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9842846963652655,
|
|
"total_bits": 924689504,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128,
|
|
"5": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9867368524796084,
|
|
"total_bits": 939164672,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.25,
|
|
0.75
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 32,
|
|
"5": 32
|
|
},
|
|
"bits": [
|
|
8,
|
|
6,
|
|
5
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.1,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9910817774944007,
|
|
"total_bits": 1069524064,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 32,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.05,
|
|
0.95
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.9919889307286787,
|
|
"total_bits": 1110384896,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.15,
|
|
0.85
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.993415660028787,
|
|
"total_bits": 1209999616,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128,
|
|
"6": 128
|
|
},
|
|
"bits": [
|
|
8,
|
|
6
|
|
],
|
|
"bits_prop": [
|
|
0.1,
|
|
0.9
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
},
|
|
{
|
|
"accuracy": 0.997326531637411,
|
|
"total_bits": 1415520512,
|
|
"gate_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"up_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
},
|
|
"down_proj": {
|
|
"group_size": {
|
|
"8": 128
|
|
},
|
|
"bits": [
|
|
8
|
|
],
|
|
"bits_prop": [
|
|
1
|
|
],
|
|
"scale_bits": 4
|
|
}
|
|
}
|
|
],
|
|
"model.norm.norm": null,
|
|
"lm_head.linear": null
|
|
},
|
|
"last_module_idx": 66
|
|
} |