Gemini Backend (#9)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-01-16 22:29:37 -08:00
parent c4707f1bb5
commit fd7c479239
13 changed files with 311 additions and 2 deletions
--- a/python/sglang/srt/models/mixtral.py
+++ b/python/sglang/srt/models/mixtral.py
@@ -355,7 +355,7 @@ class MixtralForCausalLM(nn.Module):
        ):
            if "rotary_emb.inv_freq" in name:
                continue
-            for (param_name, weight_name, shard_id) in stacked_params_mapping:
+            for param_name, weight_name, shard_id in stacked_params_mapping:
                if weight_name not in name:
                    continue
                name = name.replace(weight_name, param_name)