Support gpt-bigcode model class (#681)

2024-07-20 18:34:37 -07:00
parent 69d19188fc
commit caaad53b52
6 changed files with 341 additions and 12 deletions
--- a/python/sglang/srt/layers/logits_processor.py
+++ b/python/sglang/srt/layers/logits_processor.py
@@ -34,12 +34,11 @@ class LogitProcessorOutput:
@dataclasses.dataclass
 class LogitsMetadata:
    forward_mode: ForwardMode
-    extend_seq_lens: torch.Tensor
-    extend_start_loc: torch.Tensor
-
-    # For logprobs
    return_logprob: bool
-    top_logprobs_nums: List[int]
+
+    extend_seq_lens: torch.Tensor = None
+    extend_start_loc: torch.Tensor = None
+    top_logprobs_nums: List[int] = None

    @classmethod
    def from_input_metadata(cls, input_metadata: InputMetadata):