convert : add Llama4ForCausalLM (#16042)

* convert : add Llama4ForCausalLM * handle swa * half working version * fix use_kq_norm * fix use_kq_norm
2025-09-18 00:18:21 +07:00
parent c959b676be
commit 8f8f2274ee
4 changed files with 50 additions and 12 deletions
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -149,7 +149,7 @@ struct llama_hparams {
    bool causal_attn   = true;
    bool use_alibi     = false;
    bool attn_soft_cap = false;
-    bool use_kq_norm   = true;
+    bool use_kq_norm   = false;

    // for Classifiers
    uint32_t n_cls_out = 1;