Open AI API hidden states (#6716)

2025-06-10 17:37:29 -04:00
parent ce5ee3bdf0
commit b56de8f943
17 changed files with 606 additions and 44 deletions
--- a/test/srt/test_hidden_states.py
+++ b/test/srt/test_hidden_states.py
@@ -23,6 +23,7 @@ class TestHiddenState(CustomTestCase):
            model_path=model_path,
            random_seed=42,
            skip_tokenizer_init=True,
+            enable_return_hidden_states=True,
        )
        outputs = engine.generate(
            input_ids=input_ids,
@@ -96,6 +97,7 @@ class TestHiddenState(CustomTestCase):
            model_path=model_path,
            random_seed=42,
            skip_tokenizer_init=True,
+            enable_return_hidden_states=True,
        )
        outputs_completion_first_round = engine.generate(
            input_ids=input_ids,