初始化项目,由ModelHub XC社区提供模型
Model: scb10x/mistral-en-th-translate-v7.4-ep3 Source: Original Platform
This commit is contained in:
47
README.md
Normal file
47
README.md
Normal file
@@ -0,0 +1,47 @@
|
||||
#### Remark: This model can only translate from English to Thai, not the other way around.
|
||||
|
||||
```
|
||||
def get_translate_body_chatml(text: str):
|
||||
return {
|
||||
"prompt": f"<|im_start|> user\nI need the following text translated into Thai.\n{text} <|im_end|><|im_start|> assistant\n",
|
||||
"model": MODEL_NAME,
|
||||
# "frequency_penalty": 0.45,
|
||||
"repetition_penalty": 1.15, # this is very important, can fix all the repetitive issue with current model
|
||||
"max_tokens": 1024,
|
||||
"temperature": 0.3,
|
||||
"top_p": 0.4,
|
||||
"stop": ["<|im_end|>", "<|im_start|>"],
|
||||
}
|
||||
```
|
||||
|
||||
Example
|
||||
|
||||
1.Start vllm server
|
||||
```
|
||||
CUDA_VISIBLE_DEVICES=4 vllm serve scb10x/mistral-en-th-translate-v7.4-ep3 --port 9901
|
||||
```
|
||||
2.Inference :D
|
||||
```
|
||||
MODEL_NAME = "scb10x/mistral-en-th-translate-v7.4-ep3"
|
||||
URL = "http://localhost:9901/v1/completions" # VLLM Server
|
||||
|
||||
def send_translate_request(text: str):
|
||||
body = get_translate_body_chatml(text)
|
||||
response = requests.post(URL, json=body, headers={"Content-Type": "application/json"})
|
||||
response.raise_for_status()
|
||||
json_body = response.json()
|
||||
return json_body["choices"][0]["text"]
|
||||
|
||||
def get_translate_body_chatml(text: str):
|
||||
return {
|
||||
"prompt": f"<|im_start|> user\nI need the following text translated into Thai.\n{text} <|im_end|><|im_start|> assistant\n",
|
||||
"model": MODEL_NAME,
|
||||
"repetition_penalty": 1.15,
|
||||
"max_tokens": 2048,
|
||||
"temperature": 0.3,
|
||||
"top_p": 0.4,
|
||||
"stop": ["<|im_end|>", "<|im_start|>"],
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
Reference in New Issue
Block a user