"""Utilities for selecting and loading models.""" import contextlib from typing import Tuple, Type import torch from torch import nn from vllm.config import ModelConfig from vllm.model_executor.models import ModelRegistry @contextlib.contextmanager def set_default_torch_dtype(dtype: torch.dtype): """Sets the default torch dtype to the given dtype.""" old_dtype = torch.get_default_dtype() torch.set_default_dtype(dtype) yield torch.set_default_dtype(old_dtype) def get_model_architecture( model_config: ModelConfig) -> Tuple[Type[nn.Module], str]: architectures = getattr(model_config.hf_config, "architectures", None) or [] logger.warning("[DEBUG-ARCH] get_model_architecture: " "type(hf_config)=%s, architectures=%s, " "id(hf_config)=%s, has_text_config=%s", type(model_config.hf_config).__name__, getattr(model_config.hf_config, "architectures", "MISSING"), id(model_config.hf_config), hasattr(model_config.hf_config, "text_config")) # Special handling for quantized Mixtral. # FIXME(woosuk): This is a temporary hack. mixtral_supported = [ "fp8", "compressed-tensors", "gptq_marlin", "awq_marlin" ] if (model_config.quantization is not None and model_config.quantization not in mixtral_supported and "MixtralForCausalLM" in architectures): architectures = ["QuantMixtralForCausalLM"] return ModelRegistry.resolve_model_cls( architectures, model_path=model_config.model, revision=model_config.revision, trust_remote_code=model_config.trust_remote_code, hf_config=model_config.hf_config, ) def get_architecture_class_name(model_config: ModelConfig) -> str: return get_model_architecture(model_config)[1]