fix: draft model IMA by overide max_positional_embeddings (#10787)
Co-authored-by: Qiaolin Yu <qy254@cornell.edu>
This commit is contained in:
@@ -210,6 +210,11 @@ class ModelConfig:
|
||||
):
|
||||
logger.warning(msg)
|
||||
self.context_len = context_length
|
||||
if is_draft_model:
|
||||
self.hf_text_config.max_position_embeddings = context_length
|
||||
logger.warning(
|
||||
f"Overriding the draft model's max_position_embeddings to {context_length}."
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{msg} To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"
|
||||
|
||||
Reference in New Issue
Block a user