Open AI API hidden states (#6716)

2025-06-10 17:37:29 -04:00
parent ce5ee3bdf0
commit b56de8f943
17 changed files with 606 additions and 44 deletions
--- a/examples/runtime/hidden_states/hidden_states_engine.py
+++ b/examples/runtime/hidden_states/hidden_states_engine.py
@@ -22,6 +22,7 @@ def main():
    # Create an LLM.
    llm = sgl.Engine(
        model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+        enable_return_hidden_states=True,
    )

    sampling_params = {