EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
15
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Limit number of tokens per second for whisper. (#1958)

Browse Source 
Otherwise, it spends lots of time in the loop if the EOT token
is not predicted.
This commit is contained in:
Fangjun Kuang
2025-03-04 15:45:28 +08:00
committed by GitHub
parent 49177530ff
commit 209eaaae1d
4 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sherpa-onnx/csrc/offline-recognizer-whisper-impl.h
View File

@@ -131,7 +131,7 @@ class OfflineRecognizerWhisperImpl : public OfflineRecognizerImpl {
auto cross_kv = model_->ForwardEncoder(std::move(mel));
auto results = decoder_->Decode(std::move(cross_kv.first),
std::move(cross_kv.second));
std::move(cross_kv.second), num_frames);
auto r = Convert(results[0], symbol_table_);
s->SetResult(r);

3
sherpa-onnx/csrc/offline-whisper-decoder.h
View File

@@ -33,7 +33,8 @@ class OfflineWhisperDecoder {
* @return Return a vector of size `N` containing the decoded results.
*/
virtual std::vector<OfflineWhisperDecoderResult> Decode(
Ort::Value n_layer_cross_k, Ort::Value n_layer_cross_v) = 0;
Ort::Value n_layer_cross_k, Ort::Value n_layer_cross_v,
int32_t num_feature_frames) = 0;
virtual void SetConfig(const OfflineWhisperModelConfig &config) = 0;
};

10
sherpa-onnx/csrc/offline-whisper-greedy-search-decoder.cc
View File

@@ -19,7 +19,8 @@ void OfflineWhisperGreedySearchDecoder::SetConfig(
std::vector<OfflineWhisperDecoderResult>
OfflineWhisperGreedySearchDecoder::Decode(Ort::Value cross_k,
Ort::Value cross_v) {
Ort::Value cross_v,
int32_t num_feature_frames) {
auto memory_info =
Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault);
@@ -99,7 +100,12 @@ OfflineWhisperGreedySearchDecoder::Decode(Ort::Value cross_k,
int32_t n_text_ctx = model_->TextCtx();
std::vector<int32_t> predicted_tokens;
for (int32_t i = 0; i < n_text_ctx / 2; ++i) {
// assume at most 6 tokens per second
int32_t num_possible_tokens = num_feature_frames / 100 * 6;
num_possible_tokens = std::min<int32_t>(num_possible_tokens, n_text_ctx / 2);
for (int32_t i = 0; i < num_possible_tokens; ++i) {
if (max_token_id == model_->EOT()) {
break;
}

5
sherpa-onnx/csrc/offline-whisper-greedy-search-decoder.h
View File

@@ -18,8 +18,9 @@ class OfflineWhisperGreedySearchDecoder : public OfflineWhisperDecoder {
OfflineWhisperModel *model)
: config_(config), model_(model) {}
std::vector<OfflineWhisperDecoderResult> Decode(Ort::Value cross_k,
Ort::Value cross_v) override;
std::vector<OfflineWhisperDecoderResult> Decode(
Ort::Value cross_k, Ort::Value cross_v,
int32_t num_feature_frames) override;
void SetConfig(const OfflineWhisperModelConfig &config) override;