48 lines
1.5 KiB
Markdown
48 lines
1.5 KiB
Markdown
|
#### Remark: This model can only translate from English to Thai, not the other way around.
|
||
|
|
|
||
|
|
```
|
||
|
|
def get_translate_body_chatml(text: str):
|
||
|
|
return {
|
||
|
|
"prompt": f"<|im_start|> user\nI need the following text translated into Thai.\n{text} <|im_end|><|im_start|> assistant\n",
|
||
|
|
"model": MODEL_NAME,
|
||
|
|
# "frequency_penalty": 0.45,
|
||
|
|
"repetition_penalty": 1.15, # this is very important, can fix all the repetitive issue with current model
|
||
|
|
"max_tokens": 1024,
|
||
|
|
"temperature": 0.3,
|
||
|
|
"top_p": 0.4,
|
||
|
|
"stop": ["<|im_end|>", "<|im_start|>"],
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
Example
|
||
|
|
|
||
|
|
1.Start vllm server
|
||
|
|
```
|
||
|
|
CUDA_VISIBLE_DEVICES=4 vllm serve scb10x/mistral-en-th-translate-v7.4-ep3 --port 9901
|
||
|
|
```
|
||
|
|
2.Inference :D
|
||
|
|
```
|
||
|
|
MODEL_NAME = "scb10x/mistral-en-th-translate-v7.4-ep3"
|
||
|
|
URL = "http://localhost:9901/v1/completions" # VLLM Server
|
||
|
|
|
||
|
|
def send_translate_request(text: str):
|
||
|
|
body = get_translate_body_chatml(text)
|
||
|
|
response = requests.post(URL, json=body, headers={"Content-Type": "application/json"})
|
||
|
|
response.raise_for_status()
|
||
|
|
json_body = response.json()
|
||
|
|
return json_body["choices"][0]["text"]
|
||
|
|
|
||
|
|
def get_translate_body_chatml(text: str):
|
||
|
|
return {
|
||
|
|
"prompt": f"<|im_start|> user\nI need the following text translated into Thai.\n{text} <|im_end|><|im_start|> assistant\n",
|
||
|
|
"model": MODEL_NAME,
|
||
|
|
"repetition_penalty": 1.15,
|
||
|
|
"max_tokens": 2048,
|
||
|
|
"temperature": 0.3,
|
||
|
|
"top_p": 0.4,
|
||
|
|
"stop": ["<|im_end|>", "<|im_start|>"],
|
||
|
|
}
|
||
|
|
|
||
|
|
```
|
||
|
|
|