[feat] add small vocab table for eagle's draft model[1]. (#3822)

Co-authored-by: Achazwl <323163497@qq.com>
Co-authored-by: Chayenne <zhaochen20@outlook.com>
This commit is contained in:
Zhousx
2025-03-03 10:58:45 +08:00
committed by GitHub
parent b7e274f2d9
commit 7fbab730bd
5 changed files with 129 additions and 8 deletions

View File

@@ -128,6 +128,7 @@ class ServerArgs:
speculative_num_steps: int = 5
speculative_eagle_topk: int = 8
speculative_num_draft_tokens: int = 64
speculative_token_map: Optional[str] = None
# Double Sparsity
enable_double_sparsity: bool = False
@@ -751,6 +752,12 @@ class ServerArgs:
help="The number of token sampled from draft model in Speculative Decoding.",
default=ServerArgs.speculative_num_draft_tokens,
)
parser.add_argument(
"--speculative-token-map",
type=str,
help="The path of the draft model's small vocab table.",
default=ServerArgs.speculative_token_map,
)
# Double Sparsity
parser.add_argument(