Add OpenAI backend to the CI test (#869)

This commit is contained in:
Ying Sheng
2024-08-01 09:25:24 -07:00
committed by GitHub
parent 9e8d2c7f74
commit 4075677621
16 changed files with 30 additions and 38 deletions

View File

@@ -0,0 +1,7 @@
# Assuming the model is downdloaded at /home/ubuntu/model_weights/Llama-2-7b-chat-hf
docker run --name tgi --rm -ti --gpus all --network host \
-v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \
ghcr.io/huggingface/text-generation-inference:1.1.0 \
--model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \
--max-input-length 2048 --max-total-tokens 4096 \
--port 24000