add LogitsMetadata (#604)

2024-07-08 17:46:55 -07:00
parent f4e885b7c3
commit f25b76c02a
7 changed files with 66 additions and 40 deletions
--- a/python/sglang/srt/utils.py
+++ b/python/sglang/srt/utils.py
@@ -459,6 +459,7 @@ def monkey_patch_vllm_p2p_access_check(gpu_id: int):
    """

    import vllm.distributed.device_communicators.custom_all_reduce_utils as tgt
+
    setattr(tgt, "gpu_p2p_access_check", lambda *arg, **kwargs: True)