Fix model loading & format code (#125)

2024-01-30 23:49:52 -08:00
parent 71b54eea7d
commit ad82bac6f5
3 changed files with 16 additions and 6 deletions
--- a/python/sglang/srt/models/mixtral.py
+++ b/python/sglang/srt/models/mixtral.py
@@ -351,7 +351,11 @@ class MixtralForCausalLM(nn.Module):

        params_dict = dict(self.named_parameters())
        for name, loaded_weight in hf_model_weights_iterator(
-            model_name_or_path, cache_dir, load_format, revision
+            model_name_or_path,
+            cache_dir,
+            load_format,
+            revision,
+            fall_back_to_pt=False,
        ):
            if "rotary_emb.inv_freq" in name:
                continue