fix(build): strip \r\n from all .py files — CRLF breaks patch_ops.sh text matching
31 files had Windows line endings (\r\n) from merge commit. This causes patch_ops.sh replace_once() to fail: anchor strings use \n but file content has \r\n, so no match → patch fails → docker build fails. Also added .gitattributes to force LF for all text files going forward.
This commit is contained in:
Claude
@@ -1,39 +1,39 @@
|
||||
"""
|
||||
策略：顺序（per-sequence）fallback — 纯 PyTorch 数学实现
|
||||
==========================================================
|
||||
逐条序列用 matmul + softmax 手写 attention，完全绕开所有硬件
|
||||
flash attention kernel（ixformer / cudnnFlashAttnForward）。
|
||||
|
||||
背景：
|
||||
Iluvatar cudnnFlashAttnForward 存在两个已知问题：
|
||||
1. 不支持 is_causal=True（报错）
|
||||
2. 使用 attn_mask 路径时数值结果不正确（静默错误，输出全为"!"）
|
||||
与华为昇腾 910B4 上 llama.cpp --flash-attn off 修复同类问题的原理相同。
|
||||
纯数学路径（matmul + softmax）在任何 PyTorch 后端上结果都正确。
|
||||
|
||||
优点：
|
||||
数值正确，不依赖任何硬件特定 attention kernel。
|
||||
峰值显存 = max(seq_len)² × H × dtype_size，由 --max-model-len 控制。
|
||||
|
||||
缺点：
|
||||
并发请求的 prefill attention 串行执行。
|
||||
O(L²) 显存（无 flash attention 的 O(L) 优化）。
|
||||
|
||||
内存参考（fp16，H_local=6）：
|
||||
max-model-len=4096 → 峰值 ~200 MB
|
||||
max-model-len=8192 → 峰值 ~800 MB
|
||||
max-model-len=16384 → 峰值 ~3.2 GB
|
||||
|
||||
额外 patch（arg_utils.py）：
|
||||
vllm 0.6.3 在 max_model_len > 32K 时会自动开启 chunked prefill（无命令行
|
||||
关闭选项），原意是防止 profiling OOM。但 _run_sdpa_fallback 已通过 Q-tiling
|
||||
解决了该问题，chunked prefill 反而会把推理路径从 _run_sdpa_fallback 切换到
|
||||
_forward_prefix_pytorch，属于不必要的行为变更，因此一并禁用该自动逻辑。
|
||||
|
||||
Deploy:
|
||||
python3 modified_scripts/patch_xformers_sdpa_seq.py
|
||||
"""
|
||||
|
||||
"""
|
||||
策略：顺序（per-sequence）fallback — 纯 PyTorch 数学实现
|
||||
==========================================================
|
||||
逐条序列用 matmul + softmax 手写 attention，完全绕开所有硬件
|
||||
flash attention kernel（ixformer / cudnnFlashAttnForward）。
|
||||
|
||||
背景：
|
||||
Iluvatar cudnnFlashAttnForward 存在两个已知问题：
|
||||
1. 不支持 is_causal=True（报错）
|
||||
2. 使用 attn_mask 路径时数值结果不正确（静默错误，输出全为"!"）
|
||||
与华为昇腾 910B4 上 llama.cpp --flash-attn off 修复同类问题的原理相同。
|
||||
纯数学路径（matmul + softmax）在任何 PyTorch 后端上结果都正确。
|
||||
|
||||
优点：
|
||||
数值正确，不依赖任何硬件特定 attention kernel。
|
||||
峰值显存 = max(seq_len)² × H × dtype_size，由 --max-model-len 控制。
|
||||
|
||||
缺点：
|
||||
并发请求的 prefill attention 串行执行。
|
||||
O(L²) 显存（无 flash attention 的 O(L) 优化）。
|
||||
|
||||
内存参考（fp16，H_local=6）：
|
||||
max-model-len=4096 → 峰值 ~200 MB
|
||||
max-model-len=8192 → 峰值 ~800 MB
|
||||
max-model-len=16384 → 峰值 ~3.2 GB
|
||||
|
||||
额外 patch（arg_utils.py）：
|
||||
vllm 0.6.3 在 max_model_len > 32K 时会自动开启 chunked prefill（无命令行
|
||||
关闭选项），原意是防止 profiling OOM。但 _run_sdpa_fallback 已通过 Q-tiling
|
||||
解决了该问题，chunked prefill 反而会把推理路径从 _run_sdpa_fallback 切换到
|
||||
_forward_prefix_pytorch，属于不必要的行为变更，因此一并禁用该自动逻辑。
|
||||
|
||||
Deploy:
|
||||
python3 modified_scripts/patch_xformers_sdpa_seq.py
|
||||
"""
|
||||
|
||||
from patch_utils import package_root, replace_one_of, replace_once
|
||||
|
||||
VLLM_ROOT = package_root("vllm")
|
||||
@@ -44,24 +44,24 @@ LOGITS_PROC_PATH = (
|
||||
OUTLINES_DECODING_PATH = (
|
||||
VLLM_ROOT / "model_executor" / "guided_decoding" /
|
||||
"outlines_decoding.py")
|
||||
|
||||
# _apply_logits_processors crashes when seq_groups is None (intermediate
|
||||
# chunked-prefill chunks on the driver rank). Add an early-return guard.
|
||||
_LP_OLD_BLOCK = """\
|
||||
def _apply_logits_processors(
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
found_logits_processors = False\
|
||||
"""
|
||||
|
||||
|
||||
# _apply_logits_processors crashes when seq_groups is None (intermediate
|
||||
# chunked-prefill chunks on the driver rank). Add an early-return guard.
|
||||
_LP_OLD_BLOCK = """\
|
||||
def _apply_logits_processors(
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
found_logits_processors = False\
|
||||
"""
|
||||
|
||||
_LP_NEW_BLOCK = """\
|
||||
def _apply_logits_processors(
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if sampling_metadata.seq_groups is None: # intermediate chunked-prefill chunk
|
||||
return logits
|
||||
def _apply_logits_processors(
|
||||
logits: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if sampling_metadata.seq_groups is None: # intermediate chunked-prefill chunk
|
||||
return logits
|
||||
found_logits_processors = False\
|
||||
"""
|
||||
|
||||
@@ -116,26 +116,26 @@ _ws : JSON_WS?
|
||||
JSON_STRING: /"(\\["\\\/bfnrt]|\\u[0-9a-fA-F]{4}|[^"\\\x00-\x1f])*"/
|
||||
JSON_WS: /[ \t\r\n]{1,4}/
|
||||
%import common.SIGNED_NUMBER'''
|
||||
|
||||
# vllm 0.6.3 自动开启 chunked prefill 的原始块
|
||||
_ARG_OLD_BLOCK = """\
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter):
|
||||
self.enable_chunked_prefill = True
|
||||
logger.warning(
|
||||
"Chunked prefill is enabled by default for models with "
|
||||
"max_model_len > 32K. Currently, chunked prefill might "
|
||||
"not work with some features or models. If you "
|
||||
"encounter any issues, please disable chunked prefill "
|
||||
"by setting --enable-chunked-prefill=False.")\
|
||||
"""
|
||||
|
||||
|
||||
# vllm 0.6.3 自动开启 chunked prefill 的原始块
|
||||
_ARG_OLD_BLOCK = """\
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter):
|
||||
self.enable_chunked_prefill = True
|
||||
logger.warning(
|
||||
"Chunked prefill is enabled by default for models with "
|
||||
"max_model_len > 32K. Currently, chunked prefill might "
|
||||
"not work with some features or models. If you "
|
||||
"encounter any issues, please disable chunked prefill "
|
||||
"by setting --enable-chunked-prefill=False.")\
|
||||
"""
|
||||
|
||||
_ARG_NEW_BLOCK = """\
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter):
|
||||
pass # skip auto-enable: Q-tiling in _run_sdpa_fallback
|
||||
if (is_gpu and not use_sliding_window and not use_spec_decode
|
||||
and not self.enable_lora
|
||||
and not self.enable_prompt_adapter):
|
||||
pass # skip auto-enable: Q-tiling in _run_sdpa_fallback
|
||||
# handles long-context memory without chunked prefill\
|
||||
"""
|
||||
|
||||
@@ -163,146 +163,146 @@ _MM_PREFIX_NEW_BLOCK = """\
|
||||
"supported for multimodal models and has been disabled.")
|
||||
self.enable_prefix_caching = False\
|
||||
"""
|
||||
|
||||
FALLBACK_METHOD = '''
|
||||
def _run_sdpa_fallback(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: "XFormersMetadata",
|
||||
) -> torch.Tensor:
|
||||
"""Use ixformer flash_attn_varlen_func for head_dim > 128.
|
||||
|
||||
Verified on real BI-V100: flash_attn_func handles head_dim=256
|
||||
correctly (diff < 0.004, no NaN). For seq >= 1024, faster than
|
||||
PyTorch matmul. For profiling, sequences can be 20K+ tokens — this
|
||||
is dramatically faster than the previous Python Q-tiling fallback.
|
||||
|
||||
Falls back to pure-math if flash_attn is unavailable.
|
||||
"""
|
||||
import ixformer as _ixf
|
||||
|
||||
assert attn_metadata.seq_lens is not None
|
||||
orig_dtype = query.dtype
|
||||
num_seqs = len(attn_metadata.seq_lens)
|
||||
|
||||
q_flat = query.squeeze(0) # [T, H, D]
|
||||
k_flat = key.squeeze(0) # [T, Hkv, D]
|
||||
v_flat = value.squeeze(0)
|
||||
|
||||
# Build cu_seqlens from seq_lens
|
||||
seq_lens_list = list(attn_metadata.seq_lens)
|
||||
cu_seqlens = torch.zeros(num_seqs + 1, dtype=torch.int32,
|
||||
device=query.device)
|
||||
for i, sl in enumerate(seq_lens_list):
|
||||
cu_seqlens[i + 1] = cu_seqlens[i] + sl
|
||||
max_seqlen = max(seq_lens_list)
|
||||
|
||||
try:
|
||||
# Skip flash_attn during profiling — OOMs on large dummy batch
|
||||
import os
|
||||
if os.environ.get("BI100_IN_STARTUP_PROFILE") == "1":
|
||||
raise RuntimeError("skip flash_attn during profiling")
|
||||
out = _ixf.flash_attn_varlen_func(
|
||||
q_flat.to(torch.float16),
|
||||
k_flat.to(torch.float16),
|
||||
v_flat.to(torch.float16),
|
||||
cu_seqlens, cu_seqlens,
|
||||
max_seqlen, max_seqlen,
|
||||
causal=True,
|
||||
)
|
||||
return out.to(orig_dtype).unsqueeze(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: pure-math Q-tiling (original implementation)
|
||||
_Q_CHUNK = 256
|
||||
|
||||
# During profiling, skip expensive attention — return zeros.
|
||||
# Profiling only measures memory footprint, not output correctness.
|
||||
if os.environ.get("BI100_IN_STARTUP_PROFILE") == "1":
|
||||
return torch.zeros_like(query)
|
||||
|
||||
if (attn_metadata.query_start_loc is not None
|
||||
and len(attn_metadata.query_start_loc) == num_seqs + 1):
|
||||
q_lens = [
|
||||
int(attn_metadata.query_start_loc[i + 1].item()) -
|
||||
int(attn_metadata.query_start_loc[i].item())
|
||||
for i in range(num_seqs)
|
||||
]
|
||||
else:
|
||||
q_lens = seq_lens_list
|
||||
|
||||
output = torch.empty_like(q_flat)
|
||||
seq_start = 0
|
||||
for q_len in q_lens:
|
||||
seq_end = seq_start + q_len
|
||||
k_s = k_flat[seq_start:seq_end].permute(1, 0, 2).float()
|
||||
v_s = v_flat[seq_start:seq_end].permute(1, 0, 2).float()
|
||||
if k_s.shape[0] != self.num_heads:
|
||||
n = self.num_heads // k_s.shape[0]
|
||||
k_s = k_s.repeat_interleave(n, dim=0).contiguous()
|
||||
v_s = v_s.repeat_interleave(n, dim=0).contiguous()
|
||||
k_pos = torch.arange(q_len, device=query.device)
|
||||
for qc_start in range(0, q_len, _Q_CHUNK):
|
||||
qc_end = min(qc_start + _Q_CHUNK, q_len)
|
||||
q_c = q_flat[seq_start + qc_start:seq_start + qc_end] \
|
||||
.permute(1, 0, 2).float()
|
||||
attn_w = torch.matmul(q_c, k_s.transpose(-2, -1)) * self.scale
|
||||
qc_q_pos = torch.arange(qc_start, qc_end, device=query.device)
|
||||
mask = k_pos.unsqueeze(0) > qc_q_pos.unsqueeze(1)
|
||||
attn_w = attn_w.masked_fill(mask.unsqueeze(0), float("-inf"))
|
||||
attn_w = torch.softmax(attn_w, dim=-1)
|
||||
out_c = torch.matmul(attn_w, v_s).to(orig_dtype)
|
||||
output[seq_start + qc_start:seq_start + qc_end] = (
|
||||
out_c.permute(1, 0, 2))
|
||||
seq_start = seq_end
|
||||
return output.unsqueeze(0)
|
||||
|
||||
'''
|
||||
|
||||
OLD_XFORMER_BLOCK = """\
|
||||
self.attn_op = xops.fmha.flash.FwOp()
|
||||
if self.alibi_slopes is None:
|
||||
# Add the batch dimension.
|
||||
query = query.unsqueeze(0)
|
||||
key = key.unsqueeze(0)
|
||||
value = value.unsqueeze(0)
|
||||
out = xops.memory_efficient_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_bias=attn_bias[0],
|
||||
p=0.0,
|
||||
scale=self.scale,
|
||||
op = self.attn_op
|
||||
)
|
||||
return out.view_as(original_query)\
|
||||
"""
|
||||
|
||||
NEW_XFORMER_BLOCK = """\
|
||||
self.attn_op = xops.fmha.flash.FwOp()
|
||||
if self.alibi_slopes is None:
|
||||
# Add the batch dimension.
|
||||
query = query.unsqueeze(0)
|
||||
key = key.unsqueeze(0)
|
||||
value = value.unsqueeze(0)
|
||||
if self.head_size > 128:
|
||||
out = self._run_sdpa_fallback(query, key, value, attn_metadata)
|
||||
else:
|
||||
out = xops.memory_efficient_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_bias=attn_bias[0],
|
||||
p=0.0,
|
||||
scale=self.scale,
|
||||
op=self.attn_op,
|
||||
)
|
||||
return out.view_as(original_query)\
|
||||
"""
|
||||
|
||||
|
||||
FALLBACK_METHOD = '''
|
||||
def _run_sdpa_fallback(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: "XFormersMetadata",
|
||||
) -> torch.Tensor:
|
||||
"""Use ixformer flash_attn_varlen_func for head_dim > 128.
|
||||
|
||||
Verified on real BI-V100: flash_attn_func handles head_dim=256
|
||||
correctly (diff < 0.004, no NaN). For seq >= 1024, faster than
|
||||
PyTorch matmul. For profiling, sequences can be 20K+ tokens — this
|
||||
is dramatically faster than the previous Python Q-tiling fallback.
|
||||
|
||||
Falls back to pure-math if flash_attn is unavailable.
|
||||
"""
|
||||
import ixformer as _ixf
|
||||
|
||||
assert attn_metadata.seq_lens is not None
|
||||
orig_dtype = query.dtype
|
||||
num_seqs = len(attn_metadata.seq_lens)
|
||||
|
||||
q_flat = query.squeeze(0) # [T, H, D]
|
||||
k_flat = key.squeeze(0) # [T, Hkv, D]
|
||||
v_flat = value.squeeze(0)
|
||||
|
||||
# Build cu_seqlens from seq_lens
|
||||
seq_lens_list = list(attn_metadata.seq_lens)
|
||||
cu_seqlens = torch.zeros(num_seqs + 1, dtype=torch.int32,
|
||||
device=query.device)
|
||||
for i, sl in enumerate(seq_lens_list):
|
||||
cu_seqlens[i + 1] = cu_seqlens[i] + sl
|
||||
max_seqlen = max(seq_lens_list)
|
||||
|
||||
try:
|
||||
# Skip flash_attn during profiling — OOMs on large dummy batch
|
||||
import os
|
||||
if os.environ.get("BI100_IN_STARTUP_PROFILE") == "1":
|
||||
raise RuntimeError("skip flash_attn during profiling")
|
||||
out = _ixf.flash_attn_varlen_func(
|
||||
q_flat.to(torch.float16),
|
||||
k_flat.to(torch.float16),
|
||||
v_flat.to(torch.float16),
|
||||
cu_seqlens, cu_seqlens,
|
||||
max_seqlen, max_seqlen,
|
||||
causal=True,
|
||||
)
|
||||
return out.to(orig_dtype).unsqueeze(0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: pure-math Q-tiling (original implementation)
|
||||
_Q_CHUNK = 256
|
||||
|
||||
# During profiling, skip expensive attention — return zeros.
|
||||
# Profiling only measures memory footprint, not output correctness.
|
||||
if os.environ.get("BI100_IN_STARTUP_PROFILE") == "1":
|
||||
return torch.zeros_like(query)
|
||||
|
||||
if (attn_metadata.query_start_loc is not None
|
||||
and len(attn_metadata.query_start_loc) == num_seqs + 1):
|
||||
q_lens = [
|
||||
int(attn_metadata.query_start_loc[i + 1].item()) -
|
||||
int(attn_metadata.query_start_loc[i].item())
|
||||
for i in range(num_seqs)
|
||||
]
|
||||
else:
|
||||
q_lens = seq_lens_list
|
||||
|
||||
output = torch.empty_like(q_flat)
|
||||
seq_start = 0
|
||||
for q_len in q_lens:
|
||||
seq_end = seq_start + q_len
|
||||
k_s = k_flat[seq_start:seq_end].permute(1, 0, 2).float()
|
||||
v_s = v_flat[seq_start:seq_end].permute(1, 0, 2).float()
|
||||
if k_s.shape[0] != self.num_heads:
|
||||
n = self.num_heads // k_s.shape[0]
|
||||
k_s = k_s.repeat_interleave(n, dim=0).contiguous()
|
||||
v_s = v_s.repeat_interleave(n, dim=0).contiguous()
|
||||
k_pos = torch.arange(q_len, device=query.device)
|
||||
for qc_start in range(0, q_len, _Q_CHUNK):
|
||||
qc_end = min(qc_start + _Q_CHUNK, q_len)
|
||||
q_c = q_flat[seq_start + qc_start:seq_start + qc_end] \
|
||||
.permute(1, 0, 2).float()
|
||||
attn_w = torch.matmul(q_c, k_s.transpose(-2, -1)) * self.scale
|
||||
qc_q_pos = torch.arange(qc_start, qc_end, device=query.device)
|
||||
mask = k_pos.unsqueeze(0) > qc_q_pos.unsqueeze(1)
|
||||
attn_w = attn_w.masked_fill(mask.unsqueeze(0), float("-inf"))
|
||||
attn_w = torch.softmax(attn_w, dim=-1)
|
||||
out_c = torch.matmul(attn_w, v_s).to(orig_dtype)
|
||||
output[seq_start + qc_start:seq_start + qc_end] = (
|
||||
out_c.permute(1, 0, 2))
|
||||
seq_start = seq_end
|
||||
return output.unsqueeze(0)
|
||||
|
||||
'''
|
||||
|
||||
OLD_XFORMER_BLOCK = """\
|
||||
self.attn_op = xops.fmha.flash.FwOp()
|
||||
if self.alibi_slopes is None:
|
||||
# Add the batch dimension.
|
||||
query = query.unsqueeze(0)
|
||||
key = key.unsqueeze(0)
|
||||
value = value.unsqueeze(0)
|
||||
out = xops.memory_efficient_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_bias=attn_bias[0],
|
||||
p=0.0,
|
||||
scale=self.scale,
|
||||
op = self.attn_op
|
||||
)
|
||||
return out.view_as(original_query)\
|
||||
"""
|
||||
|
||||
NEW_XFORMER_BLOCK = """\
|
||||
self.attn_op = xops.fmha.flash.FwOp()
|
||||
if self.alibi_slopes is None:
|
||||
# Add the batch dimension.
|
||||
query = query.unsqueeze(0)
|
||||
key = key.unsqueeze(0)
|
||||
value = value.unsqueeze(0)
|
||||
if self.head_size > 128:
|
||||
out = self._run_sdpa_fallback(query, key, value, attn_metadata)
|
||||
else:
|
||||
out = xops.memory_efficient_attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_bias=attn_bias[0],
|
||||
p=0.0,
|
||||
scale=self.scale,
|
||||
op=self.attn_op,
|
||||
)
|
||||
return out.view_as(original_query)\
|
||||
"""
|
||||
|
||||
INJECT_ANCHOR = " def _run_memory_efficient_xformers_forward("
|
||||
|
||||
_PREFIX_CALL_OLD_BLOCK = """\
|
||||
@@ -367,8 +367,8 @@ def patch_file(path):
|
||||
required=True,
|
||||
already_contains=(
|
||||
"is_causal_decoder=(attn_type == AttentionType.DECODER)"))
|
||||
|
||||
|
||||
|
||||
|
||||
def patch_arg_utils(path):
|
||||
replace_once(
|
||||
path,
|
||||
@@ -382,8 +382,8 @@ def patch_arg_utils(path):
|
||||
_MM_PREFIX_NEW_BLOCK,
|
||||
required=True,
|
||||
already_contains="Keeping prefix caching enabled for the Qwen3.6")
|
||||
|
||||
|
||||
|
||||
|
||||
def patch_logits_processor(path):
|
||||
replace_once(
|
||||
path,
|
||||
@@ -402,27 +402,27 @@ def patch_outlines_json_grammar(path):
|
||||
],
|
||||
required=True,
|
||||
already_contains="JSON_WS:")
|
||||
|
||||
|
||||
def main():
|
||||
print("=== patch_xformers_sdpa_seq (sequential, pure-math) ===")
|
||||
print(f"Target: {XFORMERS_PATH}")
|
||||
patch_file(XFORMERS_PATH)
|
||||
|
||||
print("\n=== patch_arg_utils (disable chunked-prefill auto-enable) ===")
|
||||
print(f"Target: {ARG_UTILS_PATH}")
|
||||
patch_arg_utils(ARG_UTILS_PATH)
|
||||
|
||||
print("\n=== patch_logits_processor (seq_groups=None guard for chunked prefill) ===")
|
||||
|
||||
|
||||
def main():
|
||||
print("=== patch_xformers_sdpa_seq (sequential, pure-math) ===")
|
||||
print(f"Target: {XFORMERS_PATH}")
|
||||
patch_file(XFORMERS_PATH)
|
||||
|
||||
print("\n=== patch_arg_utils (disable chunked-prefill auto-enable) ===")
|
||||
print(f"Target: {ARG_UTILS_PATH}")
|
||||
patch_arg_utils(ARG_UTILS_PATH)
|
||||
|
||||
print("\n=== patch_logits_processor (seq_groups=None guard for chunked prefill) ===")
|
||||
print(f"Target: {LOGITS_PROC_PATH}")
|
||||
patch_logits_processor(LOGITS_PROC_PATH)
|
||||
|
||||
print("\n=== patch_outlines_json_grammar (reject raw control chars) ===")
|
||||
print(f"Target: {OUTLINES_DECODING_PATH}")
|
||||
patch_outlines_json_grammar(OUTLINES_DECODING_PATH)
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user