Improve logging & add logit cap (#471)

2024-05-24 03:48:53 -07:00
parent 44c998fcb5
commit 2cea6146d8
12 changed files with 106 additions and 24 deletions
--- a/benchmark/latency_throughput/test_latency.py
+++ b/benchmark/latency_throughput/test_latency.py
@@ -30,7 +30,7 @@ if __name__ == "__main__":
        response = requests.post(
            url + "/generate",
            json={
-                "text": f"{a}, ",
+                "text": f"The capital of France is",
                # "input_ids": [[2] * 256] * 196,
                "sampling_params": {
                    "temperature": 0,