Improve logging & add logit cap (#471)

2024-05-24 03:48:53 -07:00
parent 44c998fcb5
commit 2cea6146d8
12 changed files with 106 additions and 24 deletions
--- a/python/sglang/srt/utils.py
+++ b/python/sglang/srt/utils.py
@@ -92,7 +92,7 @@ def calculate_time(show=False, min_cost_ms=0.0):
    return wrapper


-def get_available_gpu_memory(gpu_id, distributed=True):
+def get_available_gpu_memory(gpu_id, distributed=False):
    """
    Get available memory for cuda:gpu_id device.
    When distributed is True, the available memory is the minimum available memory of all GPUs.