Online serving benchmarks of real datasets for hierarchical KV caching (#3211)

Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>

python/sglang/utils.py

@@ -24,10 +24,14 @@ import requests
 from IPython.display import HTML, display
 from tqdm import tqdm
 
+from sglang.srt.openai_api.protocol import ChatCompletionMessageContentPart
 from sglang.srt.utils import kill_process_tree
 
 logger = logging.getLogger(__name__)
 
+# type of content fields, can be only prompts or with images/videos
+MsgContent = Union[str, List[ChatCompletionMessageContentPart]]
+
 
 def get_exception_traceback():
     etype, value, tb = sys.exc_info()