diff --git a/computility-run.yaml b/computility-run.yaml index 9fd7ac4d..c6a03f5b 100644 --- a/computility-run.yaml +++ b/computility-run.yaml @@ -8,18 +8,18 @@ command: - --served-model-name - llm - --max-model-len - - '256000' + - '131072' - --gpu-memory-utilization - - '0.95' + - '0.90' - --trust-remote-code - -tp - '4' - --max-num-seqs - - '2' + - '1' - --disable-log-requests - --disable-frontend-multiprocessing - --max-num-batched-tokens - - '4096' + - '8192' - --enable-chunked-prefill - --max-seq-len-to-capture - '32768' @@ -45,3 +45,5 @@ env: value: admission64 - name: BI100_GDN_RESTORE_MODE value: hybrid64 + - name: BI100_MOE_COREX_TOPK_SOFTMAX + value: '0'