Compare commits
2 Commits
ca3848dae1
...
840fe923cc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
840fe923cc | ||
|
|
57a2216143 |
@@ -15,7 +15,7 @@ command:
|
||||
- -tp
|
||||
- '4'
|
||||
- --max-num-seqs
|
||||
- '1'
|
||||
- '2'
|
||||
- --disable-log-requests
|
||||
- --disable-frontend-multiprocessing
|
||||
- --enforce-eager
|
||||
|
||||
@@ -418,14 +418,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
if data.get("max_completion_tokens") is not None and data.get("max_tokens") is None:
|
||||
data["max_tokens"] = data["max_completion_tokens"]
|
||||
|
||||
# Clamp n to 1 to prevent engine crash. Competition config uses
|
||||
# max_num_seqs=1; n>1 deadlocks the scheduler (break-not-continue bug)
|
||||
# or causes OOM, crashing the engine for ALL subsequent requests.
|
||||
# Sub508: t2_n_2 → HTTP 500 → 19 cascade failures.
|
||||
# t2_n_2 will FAIL (1 choice instead of 2) but prevents cascade.
|
||||
n_val = data.get("n")
|
||||
if n_val is not None and isinstance(n_val, int) and n_val > 1:
|
||||
data["n"] = 1
|
||||
# n > max_num_seqs: clamp handled in serving_chat.py via scheduler check.
|
||||
# With max_num_seqs=2, n=2 should work. n>2 will be clamped there.
|
||||
|
||||
# Map thinking parameter → chat_template_kwargs.enable_thinking
|
||||
# OpenAI API format: thinking={"type":"enabled"} / {"type":"disabled"}
|
||||
|
||||
@@ -112,6 +112,11 @@ def _torch_chunk_gated_delta_rule(
|
||||
diagonal=0)
|
||||
|
||||
g = g.cumsum(dim=-1)
|
||||
# Clamp gate logits to prevent exp overflow → NaN cascade.
|
||||
# CCCL dispatch_reduce_deterministic.cuh: numerical stability requires
|
||||
# bounded intermediate values. Gate logit range [-20, 20] keeps exp
|
||||
# in [~2e-9, ~5e8] — safe for float32 accumulation.
|
||||
g = g.clamp(-20.0, 20.0)
|
||||
decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
|
||||
|
||||
# Lower-triangular solve WITHOUT libcusolver (not available on BI-V100).
|
||||
|
||||
Reference in New Issue
Block a user