Support LoRA in TestOpenAIVisionServer and fix fused kv_proj loading bug. (#6861)

test/srt/test_vision_openai_server_common.py

@@ -1,4 +1,5 @@
 import base64
+import copy
 import io
 import json
 import os
@@ -47,6 +48,9 @@ class TestOpenAIVisionServer(CustomTestCase):
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)
 
+    def get_request_kwargs(self):
+        return {}
+
     def test_single_image_chat_completion(self):
         client = openai.Client(api_key=self.api_key, base_url=self.base_url)
 
@@ -68,6 +72,7 @@ class TestOpenAIVisionServer(CustomTestCase):
                 },
             ],
             temperature=0,
+            **(self.get_request_kwargs()),
         )
 
         assert response.choices[0].message.role == "assistant"
@@ -130,6 +135,7 @@ class TestOpenAIVisionServer(CustomTestCase):
                 },
             ],
             temperature=0,
+            **(self.get_request_kwargs()),
         )
 
         assert response.choices[0].message.role == "assistant"
@@ -172,6 +178,7 @@ class TestOpenAIVisionServer(CustomTestCase):
                 },
             ],
             temperature=0,
+            **(self.get_request_kwargs()),
         )
 
         assert response.choices[0].message.role == "assistant"
@@ -284,6 +291,7 @@ class TestOpenAIVisionServer(CustomTestCase):
             temperature=0,
             max_tokens=1024,
             stream=False,
+            **(self.get_request_kwargs()),
         )
 
         video_response = response.choices[0].message.content
@@ -324,6 +332,9 @@ class TestOpenAIVisionServer(CustomTestCase):
             + r"""\}"""
         )
 
+        extra_kwargs = self.get_request_kwargs()
+        extra_kwargs.setdefault("extra_body", {})["regex"] = regex
+
         response = client.chat.completions.create(
             model="default",
             messages=[
@@ -342,7 +353,7 @@ class TestOpenAIVisionServer(CustomTestCase):
                 },
             ],
             temperature=0,
-            extra_body={"regex": regex},
+            **extra_kwargs,
         )
         text = response.choices[0].message.content
 
@@ -388,6 +399,7 @@ class TestOpenAIVisionServer(CustomTestCase):
                 {"role": "user", "content": content},
             ],
             temperature=0,
+            **(self.get_request_kwargs()),
         )
 
         assert response.choices[0].message.role == "assistant"
@@ -430,6 +442,7 @@ class TestOpenAIVisionServer(CustomTestCase):
             temperature=0,
             max_tokens=128,
             stream=False,
+            **(self.get_request_kwargs()),
         )
 
         audio_response = response.choices[0].message.content