27 lines
715 B
Markdown
27 lines
715 B
Markdown
---
|
|
license: mit
|
|
language:
|
|
- en
|
|
---
|
|
|
|
Inference
|
|
|
|
```bash
|
|
pip install vllm
|
|
```
|
|
|
|
```py
|
|
import os
|
|
from vllm import LLM, SamplingParams
|
|
|
|
os.environ["LD_LIBRARY_PATH"] = "/usr/local/lib/python3.12/dist-packages/nvidia/cu13/lib:" + os.environ.get("LD_LIBRARY_PATH", "")
|
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
|
|
llm = LLM(model="kaividlabs/qwen3-4b-awq", quantization="awq_marlin")
|
|
sampling_params = SamplingParams(temperature=0.7, top_p=0.8, max_tokens=512)
|
|
|
|
messages = [{"role": "user", "content": "Give me a short introduction to Formula-1"}]
|
|
prompt = llm.get_tokenizer().apply_chat_template(messages, tokenize=False, add_generation_prompt=True, enable_thinking=False)
|
|
|
|
print(outputs[0].outputs[0].text)
|
|
``` |