2024-11-17 11:23:01 +02:00
|
|
|
#ifndef GGML_METAL_IMPL
|
|
|
|
|
#define GGML_METAL_IMPL
|
|
|
|
|
|
2025-03-26 21:38:38 +02:00
|
|
|
// kernel parameters for mat-vec threadgroups
|
|
|
|
|
//
|
|
|
|
|
// N_R0: number of src0 rows to process per simdgroup
|
|
|
|
|
// N_SG: number of simdgroups per threadgroup
|
|
|
|
|
//
|
|
|
|
|
// TODO: for optimal performance, become function of the device and work size
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q4_0 4
|
|
|
|
|
#define N_SG_Q4_0 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q4_1 4
|
|
|
|
|
#define N_SG_Q4_1 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q5_0 4
|
|
|
|
|
#define N_SG_Q5_0 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q5_1 4
|
|
|
|
|
#define N_SG_Q5_1 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q8_0 4
|
|
|
|
|
#define N_SG_Q8_0 2
|
|
|
|
|
|
2025-08-05 22:10:36 +03:00
|
|
|
#define N_R0_MXFP4 2
|
|
|
|
|
#define N_SG_MXFP4 2
|
|
|
|
|
|
2025-03-26 21:38:38 +02:00
|
|
|
#define N_R0_Q2_K 4
|
|
|
|
|
#define N_SG_Q2_K 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q3_K 2
|
|
|
|
|
#define N_SG_Q3_K 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q4_K 4
|
|
|
|
|
#define N_SG_Q4_K 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q5_K 2
|
|
|
|
|
#define N_SG_Q5_K 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_Q6_K 1
|
|
|
|
|
#define N_SG_Q6_K 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ1_S 4
|
|
|
|
|
#define N_SG_IQ1_S 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ1_M 4
|
|
|
|
|
#define N_SG_IQ1_M 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ2_XXS 4
|
|
|
|
|
#define N_SG_IQ2_XXS 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ2_XS 4
|
|
|
|
|
#define N_SG_IQ2_XS 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ2_S 4
|
|
|
|
|
#define N_SG_IQ2_S 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ3_XXS 4
|
|
|
|
|
#define N_SG_IQ3_XXS 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ3_S 4
|
|
|
|
|
#define N_SG_IQ3_S 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ4_NL 2
|
|
|
|
|
#define N_SG_IQ4_NL 2
|
|
|
|
|
|
|
|
|
|
#define N_R0_IQ4_XS 2
|
|
|
|
|
#define N_SG_IQ4_XS 2
|
|
|
|
|
|
2024-11-17 11:23:01 +02:00
|
|
|
// kernel argument structs
|
|
|
|
|
//
|
|
|
|
|
// - element counters (e.g. ne00) typically use int32_t to reduce register usage
|
|
|
|
|
// however, be careful from int overflows when using those in the kernel implementation
|
|
|
|
|
//
|
|
|
|
|
// - strides (e.g. nb00) use uint64_t
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
int32_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
int32_t ne13;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int32_t ne2;
|
|
|
|
|
int32_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
int32_t dim;
|
|
|
|
|
} ggml_metal_kargs_concat;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
int32_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
int32_t ne13;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int32_t ne2;
|
|
|
|
|
int32_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
uint64_t offs;
|
2025-07-18 20:37:26 +03:00
|
|
|
uint64_t o1[8];
|
2024-11-17 11:23:01 +02:00
|
|
|
} ggml_metal_kargs_bin;
|
|
|
|
|
|
2025-08-05 22:10:36 +03:00
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
size_t nb01;
|
|
|
|
|
size_t nb02;
|
|
|
|
|
size_t nb11;
|
|
|
|
|
size_t nb21;
|
|
|
|
|
} ggml_metal_kargs_add_id;
|
|
|
|
|
|
2024-11-17 11:23:01 +02:00
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
int32_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int32_t ne2;
|
|
|
|
|
int32_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
} ggml_metal_kargs_repeat;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
int64_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
int64_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
} ggml_metal_kargs_cpy;
|
|
|
|
|
|
2024-12-04 09:19:30 +01:00
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne10;
|
|
|
|
|
int64_t ne11;
|
|
|
|
|
int64_t ne12;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
uint64_t offs;
|
|
|
|
|
bool inplace;
|
|
|
|
|
} ggml_metal_kargs_set;
|
|
|
|
|
|
2024-11-17 11:23:01 +02:00
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
int32_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int32_t ne2;
|
|
|
|
|
int32_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
int32_t n_past;
|
|
|
|
|
int32_t n_dims;
|
|
|
|
|
int32_t n_ctx_orig;
|
|
|
|
|
float freq_base;
|
|
|
|
|
float freq_scale;
|
|
|
|
|
float ext_factor;
|
|
|
|
|
float attn_factor;
|
|
|
|
|
float beta_fast;
|
|
|
|
|
float beta_slow;
|
2025-05-12 10:29:13 +02:00
|
|
|
int32_t sect_0;
|
|
|
|
|
int32_t sect_1;
|
|
|
|
|
int32_t sect_2;
|
|
|
|
|
int32_t sect_3;
|
2024-11-17 11:23:01 +02:00
|
|
|
} ggml_metal_kargs_rope;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
int32_t ne03;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne_12_2; // assume K and V are same shape
|
|
|
|
|
int32_t ne_12_3;
|
2025-03-28 20:21:59 +02:00
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
uint64_t nb21;
|
|
|
|
|
uint64_t nb22;
|
|
|
|
|
uint64_t nb23;
|
2025-06-27 21:50:57 +03:00
|
|
|
int32_t ne32;
|
2025-07-03 10:46:57 +03:00
|
|
|
int32_t ne33;
|
2024-11-17 11:23:01 +02:00
|
|
|
uint64_t nb31;
|
2025-06-27 21:50:57 +03:00
|
|
|
uint64_t nb32;
|
2025-07-03 10:46:57 +03:00
|
|
|
uint64_t nb33;
|
2024-11-17 11:23:01 +02:00
|
|
|
int32_t ne1;
|
|
|
|
|
int32_t ne2;
|
|
|
|
|
float scale;
|
|
|
|
|
float max_bias;
|
|
|
|
|
float m0;
|
|
|
|
|
float m1;
|
2025-07-18 20:37:26 +03:00
|
|
|
int32_t n_head_log2;
|
2024-11-17 11:23:01 +02:00
|
|
|
float logit_softcap;
|
|
|
|
|
} ggml_metal_kargs_flash_attn_ext;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int16_t r2;
|
|
|
|
|
int16_t r3;
|
|
|
|
|
} ggml_metal_kargs_mul_mm;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int16_t r2;
|
|
|
|
|
int16_t r3;
|
|
|
|
|
} ggml_metal_kargs_mul_mv;
|
|
|
|
|
|
2024-12-03 11:52:33 +02:00
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
int16_t r2;
|
|
|
|
|
int16_t r3;
|
|
|
|
|
int16_t nsg;
|
|
|
|
|
int16_t nxpsg;
|
|
|
|
|
int16_t r1ptg;
|
|
|
|
|
} ggml_metal_kargs_mul_mv_ext;
|
|
|
|
|
|
2024-11-17 11:23:01 +02:00
|
|
|
typedef struct {
|
2025-08-26 12:46:15 +03:00
|
|
|
int32_t ne02;
|
2025-05-09 15:14:56 +03:00
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11; // n_expert_used (bcast)
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
2025-08-26 12:46:15 +03:00
|
|
|
int32_t ne21; // n_tokens
|
2025-05-09 15:14:56 +03:00
|
|
|
int32_t ne20; // n_expert_used
|
|
|
|
|
uint64_t nb21;
|
|
|
|
|
} ggml_metal_kargs_mul_mm_id_map0;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
2024-11-17 11:23:01 +02:00
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
2025-05-09 15:14:56 +03:00
|
|
|
uint64_t nb03;
|
2025-08-26 12:46:15 +03:00
|
|
|
int32_t ne11;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int32_t ne20;
|
|
|
|
|
int32_t ne21;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
2025-05-09 15:14:56 +03:00
|
|
|
int16_t r2;
|
|
|
|
|
int16_t r3;
|
2024-11-17 11:23:01 +02:00
|
|
|
} ggml_metal_kargs_mul_mm_id;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t nei0;
|
|
|
|
|
int32_t nei1;
|
|
|
|
|
uint64_t nbi1;
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
int32_t ne13;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
int32_t ne1;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
} ggml_metal_kargs_mul_mv_id;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne00_4;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
float eps;
|
|
|
|
|
} ggml_metal_kargs_norm;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne00_4;
|
2025-07-18 20:37:26 +03:00
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
2024-11-17 11:23:01 +02:00
|
|
|
float eps;
|
2025-07-18 20:37:26 +03:00
|
|
|
int32_t nef1[3];
|
|
|
|
|
int32_t nef2[3];
|
|
|
|
|
int32_t nef3[3];
|
|
|
|
|
uint64_t nbf1[3];
|
|
|
|
|
uint64_t nbf2[3];
|
|
|
|
|
uint64_t nbf3[3];
|
2024-11-17 11:23:01 +02:00
|
|
|
} ggml_metal_kargs_rms_norm;
|
|
|
|
|
|
2025-03-18 07:27:50 +08:00
|
|
|
typedef struct {
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne00_4;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
float eps;
|
|
|
|
|
} ggml_metal_kargs_l2_norm;
|
|
|
|
|
|
2025-03-07 15:35:57 +08:00
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
int32_t n_groups;
|
|
|
|
|
float eps;
|
|
|
|
|
} ggml_metal_kargs_group_norm;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t IC;
|
|
|
|
|
int32_t IL;
|
|
|
|
|
int32_t K;
|
|
|
|
|
int32_t s0;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
} ggml_metal_kargs_conv_transpose_1d;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
uint64_t ofs0;
|
|
|
|
|
uint64_t ofs1;
|
|
|
|
|
int32_t IW;
|
|
|
|
|
int32_t IH;
|
|
|
|
|
int32_t CHW;
|
|
|
|
|
int32_t s0;
|
|
|
|
|
int32_t s1;
|
|
|
|
|
int32_t p0;
|
|
|
|
|
int32_t p1;
|
|
|
|
|
int32_t d0;
|
|
|
|
|
int32_t d1;
|
|
|
|
|
int32_t N;
|
|
|
|
|
int32_t KH;
|
|
|
|
|
int32_t KW;
|
|
|
|
|
int32_t KHW; // KH * KW, pre-computed on CPU to save GPU resources
|
|
|
|
|
} ggml_metal_kargs_im2col;
|
|
|
|
|
|
2025-06-29 11:04:10 +02:00
|
|
|
typedef struct{
|
|
|
|
|
int32_t ne00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
int32_t ne10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
int32_t ne0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
int32_t i00;
|
|
|
|
|
int32_t i10;
|
2025-08-05 22:10:36 +03:00
|
|
|
float alpha;
|
|
|
|
|
float limit;
|
2025-06-29 11:04:10 +02:00
|
|
|
} ggml_metal_kargs_glu;
|
|
|
|
|
|
2025-03-07 15:35:57 +08:00
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
int64_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int64_t ne10;
|
|
|
|
|
int64_t ne11;
|
|
|
|
|
int64_t ne12;
|
|
|
|
|
int64_t ne13;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
int64_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
} ggml_metal_kargs_sum_rows;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
2025-06-27 21:50:57 +03:00
|
|
|
int32_t ne00;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
int32_t ne02;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
int32_t ne13;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
2025-03-07 15:35:57 +08:00
|
|
|
float scale;
|
|
|
|
|
float max_bias;
|
|
|
|
|
float m0;
|
|
|
|
|
float m1;
|
2025-07-18 20:37:26 +03:00
|
|
|
int32_t n_head_log2;
|
2025-03-07 15:35:57 +08:00
|
|
|
} ggml_metal_kargs_soft_max;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int n_past;
|
|
|
|
|
} ggml_metal_kargs_diag_mask_inf;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
int64_t ne10;
|
|
|
|
|
int64_t ne11;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
} ggml_metal_kargs_ssm_conv;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t d_state;
|
|
|
|
|
int64_t d_inner;
|
llama : initial Mamba-2 support (#9126)
* llama : initial Mamba-2 support
* ggml : SIMD ggml_ssm_scan for Mamba-2
* ggml : improve ggml_mul speed when masking recurrent states
* llama : support running Mamba-Codestral-7B-v0.1
* llama : fix Mamba-2 conv state saving
* ggml : make the ggml_mul fast broadcast path more consistently formatted
* llama : remove unused variable
* llama : add missing break
* convert_hf : prefer SentencePiece tokenizer for Mamba-2 when present
The tokenzier.json of Mamba-Codestral-7B-v0.1 otherwise requires
workarounds to work correctly.
* llama : avoid redundant state copy for Mamba 1 and 2
* metal : attempt to adapt SSM_SCAN for Mamba-2
* metal : fix SSM_SCAN pipeline scope
* metal : use log and exp instead of log1pf and expf in SSM_SCAN
* metal : remove unused arguments for SSM_SCAN
The max index is 31, so trimming the arguments is necessary.
* metal : add back n_seqs to SSM_SCAN args
Whoops, this is needed for the offset in the concatenated output.
* metal : fix SSM_SCAN state head offset
* metal : fix wrong number of tokens per sequence in SSM_SCAN
* ggml : remove unused fast broadcast path in GGML_MUL
This was initially added because states were masked with ggml_mul,
but this is no longer done and so this "optimisation" is no longer
necessary, or at least not worth the additional code complexity.
* ggml : avoid multiply by D in GGML_OP_SSM_SCAN
This makes the weight buft detection in src/llama.cpp simpler.
* convert : transpose Mamba-2 A, D and reshape SSM_NORM
This breaks existing conversions of Mamba-2 models
to avoid some reshapes.
Not sure if it's a good idea,
but it makes the graph slightly cleaner.
* llama : more appropriate SSM_SCAN and SSM_CONV buft support checks
* convert : fix flake8 lint
* metal : fix confusion between ; and ,
* metal : add missing args for nb references in ssm_scan_f32_group
* metal : single-user mamba2 inference works
* kv-cache : remove const_cast when setting inputs for s_copy
And also fix multi-user inference for recurrent models
by using cell_id instead of i as the kv cell index
when populating s_copy.
* convert : avoid AutoConfig for Mamba and Mamba2 hparams
* kv-cache : allow context shift for recurrent models
* graph : fix recurrent state copies when avoiding copies
Works, but using lambda functions might not be that clean.
* ggml : fix mamba2 ssm scan when compiled with SVE
* ggml-cpu : reorder SVE FMA for consistency with other SIMD arches
* cuda : implement ssm scan for Mamba2
There is still room for improvement, but it works!
* cuda : adapt Mamba1 ssm scan to shape changes from Mamba2
* mamba : fix mismatched new and delete size for llm_build_mamba
Subclasses of llm_graph_context cannot have extra fields,
because the called destructor is not the one from the subclass.
This otherwise would cause problems when runnning Mamba-(1|2) inference
when compiled -DGGML_SANITIZE_ADDRESS=ON
* cuda : graceful fallback for Mamba-1 models with weird embd size
2025-07-02 13:10:24 -04:00
|
|
|
int64_t n_head;
|
|
|
|
|
int64_t n_group;
|
2025-03-07 15:35:57 +08:00
|
|
|
int64_t n_seq_tokens;
|
|
|
|
|
int64_t n_seqs;
|
2025-07-25 10:47:39 -06:00
|
|
|
int64_t s_off;
|
2025-03-07 15:35:57 +08:00
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
llama : initial Mamba-2 support (#9126)
* llama : initial Mamba-2 support
* ggml : SIMD ggml_ssm_scan for Mamba-2
* ggml : improve ggml_mul speed when masking recurrent states
* llama : support running Mamba-Codestral-7B-v0.1
* llama : fix Mamba-2 conv state saving
* ggml : make the ggml_mul fast broadcast path more consistently formatted
* llama : remove unused variable
* llama : add missing break
* convert_hf : prefer SentencePiece tokenizer for Mamba-2 when present
The tokenzier.json of Mamba-Codestral-7B-v0.1 otherwise requires
workarounds to work correctly.
* llama : avoid redundant state copy for Mamba 1 and 2
* metal : attempt to adapt SSM_SCAN for Mamba-2
* metal : fix SSM_SCAN pipeline scope
* metal : use log and exp instead of log1pf and expf in SSM_SCAN
* metal : remove unused arguments for SSM_SCAN
The max index is 31, so trimming the arguments is necessary.
* metal : add back n_seqs to SSM_SCAN args
Whoops, this is needed for the offset in the concatenated output.
* metal : fix SSM_SCAN state head offset
* metal : fix wrong number of tokens per sequence in SSM_SCAN
* ggml : remove unused fast broadcast path in GGML_MUL
This was initially added because states were masked with ggml_mul,
but this is no longer done and so this "optimisation" is no longer
necessary, or at least not worth the additional code complexity.
* ggml : avoid multiply by D in GGML_OP_SSM_SCAN
This makes the weight buft detection in src/llama.cpp simpler.
* convert : transpose Mamba-2 A, D and reshape SSM_NORM
This breaks existing conversions of Mamba-2 models
to avoid some reshapes.
Not sure if it's a good idea,
but it makes the graph slightly cleaner.
* llama : more appropriate SSM_SCAN and SSM_CONV buft support checks
* convert : fix flake8 lint
* metal : fix confusion between ; and ,
* metal : add missing args for nb references in ssm_scan_f32_group
* metal : single-user mamba2 inference works
* kv-cache : remove const_cast when setting inputs for s_copy
And also fix multi-user inference for recurrent models
by using cell_id instead of i as the kv cell index
when populating s_copy.
* convert : avoid AutoConfig for Mamba and Mamba2 hparams
* kv-cache : allow context shift for recurrent models
* graph : fix recurrent state copies when avoiding copies
Works, but using lambda functions might not be that clean.
* ggml : fix mamba2 ssm scan when compiled with SVE
* ggml-cpu : reorder SVE FMA for consistency with other SIMD arches
* cuda : implement ssm scan for Mamba2
There is still room for improvement, but it works!
* cuda : adapt Mamba1 ssm scan to shape changes from Mamba2
* mamba : fix mismatched new and delete size for llm_build_mamba
Subclasses of llm_graph_context cannot have extra fields,
because the called destructor is not the one from the subclass.
This otherwise would cause problems when runnning Mamba-(1|2) inference
when compiled -DGGML_SANITIZE_ADDRESS=ON
* cuda : graceful fallback for Mamba-1 models with weird embd size
2025-07-02 13:10:24 -04:00
|
|
|
uint64_t nb03;
|
2025-03-07 15:35:57 +08:00
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb13;
|
|
|
|
|
uint64_t nb21;
|
|
|
|
|
uint64_t nb22;
|
|
|
|
|
uint64_t nb31;
|
|
|
|
|
uint64_t nb41;
|
|
|
|
|
uint64_t nb42;
|
llama : initial Mamba-2 support (#9126)
* llama : initial Mamba-2 support
* ggml : SIMD ggml_ssm_scan for Mamba-2
* ggml : improve ggml_mul speed when masking recurrent states
* llama : support running Mamba-Codestral-7B-v0.1
* llama : fix Mamba-2 conv state saving
* ggml : make the ggml_mul fast broadcast path more consistently formatted
* llama : remove unused variable
* llama : add missing break
* convert_hf : prefer SentencePiece tokenizer for Mamba-2 when present
The tokenzier.json of Mamba-Codestral-7B-v0.1 otherwise requires
workarounds to work correctly.
* llama : avoid redundant state copy for Mamba 1 and 2
* metal : attempt to adapt SSM_SCAN for Mamba-2
* metal : fix SSM_SCAN pipeline scope
* metal : use log and exp instead of log1pf and expf in SSM_SCAN
* metal : remove unused arguments for SSM_SCAN
The max index is 31, so trimming the arguments is necessary.
* metal : add back n_seqs to SSM_SCAN args
Whoops, this is needed for the offset in the concatenated output.
* metal : fix SSM_SCAN state head offset
* metal : fix wrong number of tokens per sequence in SSM_SCAN
* ggml : remove unused fast broadcast path in GGML_MUL
This was initially added because states were masked with ggml_mul,
but this is no longer done and so this "optimisation" is no longer
necessary, or at least not worth the additional code complexity.
* ggml : avoid multiply by D in GGML_OP_SSM_SCAN
This makes the weight buft detection in src/llama.cpp simpler.
* convert : transpose Mamba-2 A, D and reshape SSM_NORM
This breaks existing conversions of Mamba-2 models
to avoid some reshapes.
Not sure if it's a good idea,
but it makes the graph slightly cleaner.
* llama : more appropriate SSM_SCAN and SSM_CONV buft support checks
* convert : fix flake8 lint
* metal : fix confusion between ; and ,
* metal : add missing args for nb references in ssm_scan_f32_group
* metal : single-user mamba2 inference works
* kv-cache : remove const_cast when setting inputs for s_copy
And also fix multi-user inference for recurrent models
by using cell_id instead of i as the kv cell index
when populating s_copy.
* convert : avoid AutoConfig for Mamba and Mamba2 hparams
* kv-cache : allow context shift for recurrent models
* graph : fix recurrent state copies when avoiding copies
Works, but using lambda functions might not be that clean.
* ggml : fix mamba2 ssm scan when compiled with SVE
* ggml-cpu : reorder SVE FMA for consistency with other SIMD arches
* cuda : implement ssm scan for Mamba2
There is still room for improvement, but it works!
* cuda : adapt Mamba1 ssm scan to shape changes from Mamba2
* mamba : fix mismatched new and delete size for llm_build_mamba
Subclasses of llm_graph_context cannot have extra fields,
because the called destructor is not the one from the subclass.
This otherwise would cause problems when runnning Mamba-(1|2) inference
when compiled -DGGML_SANITIZE_ADDRESS=ON
* cuda : graceful fallback for Mamba-1 models with weird embd size
2025-07-02 13:10:24 -04:00
|
|
|
uint64_t nb43;
|
2025-03-07 15:35:57 +08:00
|
|
|
uint64_t nb51;
|
|
|
|
|
uint64_t nb52;
|
llama : initial Mamba-2 support (#9126)
* llama : initial Mamba-2 support
* ggml : SIMD ggml_ssm_scan for Mamba-2
* ggml : improve ggml_mul speed when masking recurrent states
* llama : support running Mamba-Codestral-7B-v0.1
* llama : fix Mamba-2 conv state saving
* ggml : make the ggml_mul fast broadcast path more consistently formatted
* llama : remove unused variable
* llama : add missing break
* convert_hf : prefer SentencePiece tokenizer for Mamba-2 when present
The tokenzier.json of Mamba-Codestral-7B-v0.1 otherwise requires
workarounds to work correctly.
* llama : avoid redundant state copy for Mamba 1 and 2
* metal : attempt to adapt SSM_SCAN for Mamba-2
* metal : fix SSM_SCAN pipeline scope
* metal : use log and exp instead of log1pf and expf in SSM_SCAN
* metal : remove unused arguments for SSM_SCAN
The max index is 31, so trimming the arguments is necessary.
* metal : add back n_seqs to SSM_SCAN args
Whoops, this is needed for the offset in the concatenated output.
* metal : fix SSM_SCAN state head offset
* metal : fix wrong number of tokens per sequence in SSM_SCAN
* ggml : remove unused fast broadcast path in GGML_MUL
This was initially added because states were masked with ggml_mul,
but this is no longer done and so this "optimisation" is no longer
necessary, or at least not worth the additional code complexity.
* ggml : avoid multiply by D in GGML_OP_SSM_SCAN
This makes the weight buft detection in src/llama.cpp simpler.
* convert : transpose Mamba-2 A, D and reshape SSM_NORM
This breaks existing conversions of Mamba-2 models
to avoid some reshapes.
Not sure if it's a good idea,
but it makes the graph slightly cleaner.
* llama : more appropriate SSM_SCAN and SSM_CONV buft support checks
* convert : fix flake8 lint
* metal : fix confusion between ; and ,
* metal : add missing args for nb references in ssm_scan_f32_group
* metal : single-user mamba2 inference works
* kv-cache : remove const_cast when setting inputs for s_copy
And also fix multi-user inference for recurrent models
by using cell_id instead of i as the kv cell index
when populating s_copy.
* convert : avoid AutoConfig for Mamba and Mamba2 hparams
* kv-cache : allow context shift for recurrent models
* graph : fix recurrent state copies when avoiding copies
Works, but using lambda functions might not be that clean.
* ggml : fix mamba2 ssm scan when compiled with SVE
* ggml-cpu : reorder SVE FMA for consistency with other SIMD arches
* cuda : implement ssm scan for Mamba2
There is still room for improvement, but it works!
* cuda : adapt Mamba1 ssm scan to shape changes from Mamba2
* mamba : fix mismatched new and delete size for llm_build_mamba
Subclasses of llm_graph_context cannot have extra fields,
because the called destructor is not the one from the subclass.
This otherwise would cause problems when runnning Mamba-(1|2) inference
when compiled -DGGML_SANITIZE_ADDRESS=ON
* cuda : graceful fallback for Mamba-1 models with weird embd size
2025-07-02 13:10:24 -04:00
|
|
|
uint64_t nb53;
|
2025-03-07 15:35:57 +08:00
|
|
|
} ggml_metal_kargs_ssm_scan;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
int64_t ne10;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
} ggml_metal_kargs_get_rows;
|
|
|
|
|
|
2025-06-27 16:41:40 +03:00
|
|
|
typedef struct {
|
|
|
|
|
int32_t nk0;
|
|
|
|
|
int32_t ne01;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int32_t ne11;
|
|
|
|
|
int32_t ne12;
|
|
|
|
|
uint64_t nb10;
|
|
|
|
|
uint64_t nb11;
|
|
|
|
|
uint64_t nb12;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
} ggml_metal_kargs_set_rows;
|
|
|
|
|
|
2025-03-07 15:35:57 +08:00
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
int64_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
int64_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
float sf0;
|
|
|
|
|
float sf1;
|
|
|
|
|
float sf2;
|
|
|
|
|
float sf3;
|
|
|
|
|
} ggml_metal_kargs_upscale;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
int64_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
int64_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
} ggml_metal_kargs_pad;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne00;
|
|
|
|
|
int64_t ne01;
|
|
|
|
|
int64_t ne02;
|
|
|
|
|
int64_t ne03;
|
|
|
|
|
uint64_t nb00;
|
|
|
|
|
uint64_t nb01;
|
|
|
|
|
uint64_t nb02;
|
|
|
|
|
uint64_t nb03;
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
int64_t ne1;
|
|
|
|
|
int64_t ne2;
|
|
|
|
|
int64_t ne3;
|
|
|
|
|
uint64_t nb0;
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
uint64_t nb2;
|
|
|
|
|
uint64_t nb3;
|
|
|
|
|
int32_t p0;
|
|
|
|
|
int32_t p1;
|
|
|
|
|
} ggml_metal_kargs_pad_reflect_1d;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
uint64_t nb1;
|
|
|
|
|
int dim;
|
|
|
|
|
int max_period;
|
|
|
|
|
} ggml_metal_kargs_timestep_embedding;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
float slope;
|
|
|
|
|
} ggml_metal_kargs_leaky_relu;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ncols;
|
|
|
|
|
int64_t ncols_pad;
|
|
|
|
|
} ggml_metal_kargs_argsort;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int64_t ne0;
|
|
|
|
|
float start;
|
|
|
|
|
float step;
|
|
|
|
|
} ggml_metal_kargs_arange;
|
|
|
|
|
|
|
|
|
|
typedef struct {
|
|
|
|
|
int32_t k0;
|
|
|
|
|
int32_t k1;
|
|
|
|
|
int32_t s0;
|
|
|
|
|
int32_t s1;
|
|
|
|
|
int32_t p0;
|
|
|
|
|
int32_t p1;
|
|
|
|
|
int64_t IH;
|
|
|
|
|
int64_t IW;
|
|
|
|
|
int64_t OH;
|
|
|
|
|
int64_t OW;
|
|
|
|
|
int64_t parallel_elements;
|
|
|
|
|
} ggml_metal_kargs_pool_2d;
|
|
|
|
|
|
2024-11-17 11:23:01 +02:00
|
|
|
#endif // GGML_METAL_IMPL
|