ef6abf3dc7
Adds ALL files needed for Dockerfile build:
- qwen3_6_scripts/ (baseline patches + our optimizations)
- vllm/ (full vllm package)
- paged_attention_v2_pytorch.py (V2 with single-bmm optimization)
- Dockerfile + computility-run.yaml
Our optimizations vs baseline:
1. paged_attn.py: pre-gathered context KV (eliminates 194 gather calls),
Triton try/fallback, V2 heuristic, threshold 32K→64K
2. paged_attention_v2_pytorch.py: fills NotImplementedError,
single-bmm Phase 1 (195 launches → 3)
3. patch_enable_triton.py: HAS_TRITON=True with safety fallback
4. patch_triton_tuning.py: BLOCK=64, NUM_WARPS=4 for BI-V100
5. computility-run.yaml: gpu-memory-utilization 0.9→0.95,
max-num-batched-tokens 8192→16384
This repo can now be submitted to dev.modelhub.org.cn as-is.
88 lines
2.8 KiB
Python
88 lines
2.8 KiB
Python
def find_common_prefix(s1: str, s2: str) -> str:
|
|
"""
|
|
Finds a common prefix that is shared between two strings, if there is one.
|
|
Order of arguments is NOT important.
|
|
|
|
This function is provided as a UTILITY for extracting information from JSON
|
|
generated by partial_json_parser, to help in ensuring that the right tokens
|
|
are returned in streaming, so that close-quotes, close-brackets and
|
|
close-braces are not returned prematurely.
|
|
|
|
e.g. find_common_prefix('{"fruit": "ap"}', '{"fruit": "apple"}') ->
|
|
'{"fruit": "ap'
|
|
"""
|
|
prefix = ''
|
|
min_length = min(len(s1), len(s2))
|
|
for i in range(0, min_length):
|
|
if s1[i] == s2[i]:
|
|
prefix += s1[i]
|
|
else:
|
|
break
|
|
return prefix
|
|
|
|
|
|
def find_common_suffix(s1: str, s2: str) -> str:
|
|
"""
|
|
Finds a common suffix shared between two strings, if there is one. Order of
|
|
arguments is NOT important.
|
|
Stops when the suffix ends OR it hits an alphanumeric character
|
|
|
|
e.g. find_common_suffix('{"fruit": "ap"}', '{"fruit": "apple"}') -> '"}'
|
|
"""
|
|
suffix = ''
|
|
min_length = min(len(s1), len(s2))
|
|
for i in range(1, min_length + 1):
|
|
if s1[-i] == s2[-i] and not s1[-i].isalnum():
|
|
suffix = s1[-i] + suffix
|
|
else:
|
|
break
|
|
return suffix
|
|
|
|
|
|
def extract_intermediate_diff(curr: str, old: str) -> str:
|
|
"""
|
|
Given two strings, extract the difference in the middle between two strings
|
|
that are known to have a common prefix and/or suffix.
|
|
|
|
This function is provided as a UTILITY for extracting information from JSON
|
|
generated by partial_json_parser, to help in ensuring that the right tokens
|
|
are returned in streaming, so that close-quotes, close-brackets and
|
|
close-braces are not returned prematurely. The order of arguments IS
|
|
important - the new version of the partially-parsed JSON must be the first
|
|
argument, and the secnod argument must be from the previous generation.
|
|
|
|
What it returns, is tokens that should be streamed to the client.
|
|
|
|
e.g. extract_intermediate_diff('{"fruit": "apple"}', '{"fruit": "ap"}')
|
|
-> 'ple'
|
|
|
|
"""
|
|
suffix = find_common_suffix(curr, old)
|
|
|
|
old = old[::-1].replace(suffix[::-1], '', 1)[::-1]
|
|
prefix = find_common_prefix(curr, old)
|
|
diff = curr
|
|
if len(suffix):
|
|
diff = diff[::-1].replace(suffix[::-1], '', 1)[::-1]
|
|
|
|
if len(prefix):
|
|
# replace the prefix only once in case it's mirrored
|
|
diff = diff.replace(prefix, '', 1)
|
|
|
|
return diff
|
|
|
|
|
|
def find_all_indices(string, substring):
|
|
"""
|
|
Find all (starting) indices of a substring in a given string. Useful for
|
|
tool call extraction
|
|
"""
|
|
indices = []
|
|
index = -1
|
|
while True:
|
|
index = string.find(substring, index + 1)
|
|
if index == -1:
|
|
break
|
|
indices.append(index)
|
|
return indices
|