Return logprob for choices (#87)

2024-01-23 05:07:30 -08:00
parent 9e037c822c
commit 9a16fea012
15 changed files with 161 additions and 112 deletions
--- a/test/srt/test_httpserver_decode.py
+++ b/test/srt/test_httpserver_decode.py
@@ -26,6 +26,8 @@ if __name__ == "__main__":
                "temperature": 0,
                "max_new_tokens": 32,
            },
+            # "return_logprob": True,
+            # "logprob_start_len": 0,
        },
    )
    print(response.json())