Tiny add logging for GC (#7406)
This commit is contained in:
@@ -140,6 +140,7 @@ from sglang.srt.utils import (
|
|||||||
DeepEPMode,
|
DeepEPMode,
|
||||||
DynamicGradMode,
|
DynamicGradMode,
|
||||||
broadcast_pyobj,
|
broadcast_pyobj,
|
||||||
|
configure_gc_logger,
|
||||||
configure_logger,
|
configure_logger,
|
||||||
disable_request_logging,
|
disable_request_logging,
|
||||||
get_available_gpu_memory,
|
get_available_gpu_memory,
|
||||||
@@ -508,6 +509,9 @@ class Scheduler(
|
|||||||
)
|
)
|
||||||
self.init_disaggregation()
|
self.init_disaggregation()
|
||||||
|
|
||||||
|
if get_bool_env_var("SGLANG_GC_LOG"):
|
||||||
|
configure_gc_logger()
|
||||||
|
|
||||||
def maybe_sleep_on_idle(self):
|
def maybe_sleep_on_idle(self):
|
||||||
if self.idle_sleeper is not None:
|
if self.idle_sleeper is not None:
|
||||||
self.idle_sleeper.maybe_sleep()
|
self.idle_sleeper.maybe_sleep()
|
||||||
|
|||||||
@@ -2485,3 +2485,28 @@ def dynamic_import(func_path: str):
|
|||||||
module = importlib.import_module(module_path)
|
module = importlib.import_module(module_path)
|
||||||
func = getattr(module, func_name)
|
func = getattr(module, func_name)
|
||||||
return func
|
return func
|
||||||
|
|
||||||
|
|
||||||
|
def configure_gc_logger():
|
||||||
|
logger.info("Enable GC Logger")
|
||||||
|
|
||||||
|
import gc
|
||||||
|
|
||||||
|
gc_start_time = {}
|
||||||
|
|
||||||
|
def gc_callback(phase, info):
|
||||||
|
gen = info.get("generation", "?")
|
||||||
|
if phase == "start":
|
||||||
|
gc_start_time[gen] = time.time()
|
||||||
|
logger.info(f"GC start: Time {time.time()} | Generation {gen}")
|
||||||
|
elif phase == "stop":
|
||||||
|
duration = time.time() - gc_start_time.get(gen, time.time())
|
||||||
|
collected = info.get("collected", "?")
|
||||||
|
uncollectable = info.get("uncollectable", "?")
|
||||||
|
logger.info(
|
||||||
|
f"GC end: Time {time.time()} | Generation {gen} | "
|
||||||
|
f"Duration: {duration:.4f}s | Collected: {collected} | Uncollectable: {uncollectable} "
|
||||||
|
f'{"(LONG GC)" if duration > 0.1 else ""}'
|
||||||
|
)
|
||||||
|
|
||||||
|
gc.callbacks.append(gc_callback)
|
||||||
|
|||||||
Reference in New Issue
Block a user