[Feat] Add window attention for gemma-2 (#1056)

2024-08-13 17:01:26 -07:00
parent ad3e4f1619
commit 0909bb0d2f
11 changed files with 320 additions and 127 deletions
--- a/python/sglang/bench_latency.py
+++ b/python/sglang/bench_latency.py
@@ -64,7 +64,7 @@ class BenchArgs:
    run_name: str = "before"
    batch_size: Tuple[int] = (1,)
    input_len: Tuple[int] = (1024,)
-    output_len: Tuple[int] = (4,)
+    output_len: Tuple[int] = (16,)
    result_filename: str = ""
    correctness_test: bool = False
    # This is only used for correctness test