[feat] add small vocab table for eagle's draft model[1]. (#3822)
Co-authored-by: Achazwl <323163497@qq.com> Co-authored-by: Chayenne <zhaochen20@outlook.com>
This commit is contained in:
@@ -128,6 +128,7 @@ class ServerArgs:
|
||||
speculative_num_steps: int = 5
|
||||
speculative_eagle_topk: int = 8
|
||||
speculative_num_draft_tokens: int = 64
|
||||
speculative_token_map: Optional[str] = None
|
||||
|
||||
# Double Sparsity
|
||||
enable_double_sparsity: bool = False
|
||||
@@ -751,6 +752,12 @@ class ServerArgs:
|
||||
help="The number of token sampled from draft model in Speculative Decoding.",
|
||||
default=ServerArgs.speculative_num_draft_tokens,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speculative-token-map",
|
||||
type=str,
|
||||
help="The path of the draft model's small vocab table.",
|
||||
default=ServerArgs.speculative_token_map,
|
||||
)
|
||||
|
||||
# Double Sparsity
|
||||
parser.add_argument(
|
||||
|
||||
Reference in New Issue
Block a user