Support page size > 1 (#4356)

2025-03-12 22:22:39 -07:00
parent 2f6bacee03
commit c76040e31b
23 changed files with 877 additions and 284 deletions
--- a/test/srt/test_gptqmodel_dynamic.py
+++ b/test/srt/test_gptqmodel_dynamic.py
@@ -184,6 +184,7 @@ class TestGPTQModelDynamicWithMarlin(unittest.TestCase):
                "text": "The capital of France is",
                "sampling_params": {
                    "max_new_tokens": max_new_tokens,
+                    "temperature": 0.001,
                },
            },
        )