openai chat speculative execution (#250)

Co-authored-by: Ying Sheng <sqy1415@gmail.com>
2024-05-18 22:23:53 -07:00
parent 5b647543c1
commit ec380dfd30
11 changed files with 316 additions and 45 deletions
--- a/python/sglang/srt/managers/tokenizer_manager.py
+++ b/python/sglang/srt/managers/tokenizer_manager.py
@@ -341,7 +341,6 @@ class TokenizerManager:
        return top_logprobs


-
 global global_processor


@@ -385,4 +384,4 @@ def get_pixel_values(
            pixel_values = pixel_values.astype(np.float16)
            return pixel_values, image_hash, image.size
    except Exception:
-        print("Exception in TokenizerManager:\n" + get_exception_traceback())
+        print("Exception in TokenizerManager:\n" + get_exception_traceback())