初始化项目,由ModelHub XC社区提供模型
Model: RightNowAI/RightNow-Arabic-0.5B-Turbo Source: Original Platform
This commit is contained in:
11
inference.json
Normal file
11
inference.json
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"_method": "HF generate(), greedy, attn_implementation=sdpa, no repetition penalty, no n-gram blocking",
|
||||
"_note": "vLLM/TGI/SGLang typically achieve 2-5x higher tok/s at small batches due to CUDA graphs and optimized kernels. Use vLLM for production deployment.",
|
||||
"bs1_tok_per_sec": 82,
|
||||
"bs4_tok_per_sec": 323,
|
||||
"bs8_tok_per_sec": 618,
|
||||
"bs32_tok_per_sec": 2154,
|
||||
"ttft_ms_bs1": 14,
|
||||
"peak_mem_gb_bf16": 1.17,
|
||||
"peak_mem_gb_int8": 0.66
|
||||
}
|
||||
Reference in New Issue
Block a user