Allow input_ids in the input of the /generate endpoint (#363)

2024-05-12 12:29:00 -10:00
parent 6e09cf6a15
commit 04c0b21488
5 changed files with 43 additions and 10 deletions
--- a/benchmark/latency_throughput/test_latency.py
+++ b/benchmark/latency_throughput/test_latency.py
@@ -30,7 +30,7 @@ if __name__ == "__main__":
        response = requests.post(
            url + "/generate",
            json={
-                "text": f"{a}, ",
+                "input_ids": [[1,2,3], [1,2,3]],
                "sampling_params": {
                    "temperature": 0,
                    "max_new_tokens": max_new_tokens,