Minor fix in compiler & format (#545)

This commit is contained in:
sglang
2024-06-29 23:42:14 -07:00
committed by GitHub
parent 9ce89bc14b
commit 11616fc6bd
12 changed files with 28 additions and 33 deletions

View File

@@ -4,7 +4,7 @@ from queue import Queue
from typing import List, Union
from sglang.global_config import global_config
from sglang.lang.interpreter import ProgramState, StreamExecutor, pin_program
from sglang.lang.interpreter import ProgramState, StreamExecutor, cache_program
from sglang.lang.ir import (
SglArgument,
SglConstantText,
@@ -184,7 +184,7 @@ class CompiledFunction:
# Extract prefix by tracing and cache it
if len(batch_kwargs) > 1:
pin_program(self.function, backend)
cache_program(self.function, backend)
# Run all programs
if num_threads == "auto":