add LogitsMetadata (#604)

2024-07-08 17:46:55 -07:00
parent f4e885b7c3
commit f25b76c02a
7 changed files with 66 additions and 40 deletions
--- a/python/sglang/srt/layers/radix_attention.py
+++ b/python/sglang/srt/layers/radix_attention.py
@@ -2,9 +2,8 @@

 import numpy as np
 import torch
-from torch import nn
-
 from flashinfer.cascade import merge_state
+from torch import nn

 from sglang.global_config import global_config
 from sglang.srt.layers.extend_attention import extend_attention_fwd