From aa7108729bd07e773f03ea1a49eb8bb27367ff8b Mon Sep 17 00:00:00 2001 From: PF Luo Date: Wed, 26 Apr 2023 11:41:04 +0800 Subject: [PATCH] share GetHypsRowSplits interface and fix getting Topk not taking logprob (#131) --- sherpa-onnx/csrc/hypothesis.cc | 15 +++++ sherpa-onnx/csrc/hypothesis.h | 3 + ...transducer-modified-beam-search-decoder.cc | 15 ----- ...transducer-modified-beam-search-decoder.cc | 57 +++++++++++-------- 4 files changed, 52 insertions(+), 38 deletions(-) diff --git a/sherpa-onnx/csrc/hypothesis.cc b/sherpa-onnx/csrc/hypothesis.cc index 5c2693d7..f07a161c 100644 --- a/sherpa-onnx/csrc/hypothesis.cc +++ b/sherpa-onnx/csrc/hypothesis.cc @@ -66,4 +66,19 @@ std::vector Hypotheses::GetTopK(int32_t k, bool length_norm) const { return {all_hyps.begin(), all_hyps.begin() + k}; } +const std::vector GetHypsRowSplits( + const std::vector &hyps) { + std::vector row_splits; + row_splits.reserve(hyps.size() + 1); + + row_splits.push_back(0); + int32_t s = 0; + for (const auto &h : hyps) { + s += h.Size(); + row_splits.push_back(s); + } + + return row_splits; +} + } // namespace sherpa_onnx diff --git a/sherpa-onnx/csrc/hypothesis.h b/sherpa-onnx/csrc/hypothesis.h index 86221729..198e40a7 100644 --- a/sherpa-onnx/csrc/hypothesis.h +++ b/sherpa-onnx/csrc/hypothesis.h @@ -121,6 +121,9 @@ class Hypotheses { Map hyps_dict_; }; +const std::vector GetHypsRowSplits( + const std::vector &hyps); + } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_HYPOTHESIS_H_ diff --git a/sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.cc b/sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.cc index c4aa2a75..d9660684 100644 --- a/sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.cc +++ b/sherpa-onnx/csrc/offline-transducer-modified-beam-search-decoder.cc @@ -15,21 +15,6 @@ namespace sherpa_onnx { -static std::vector GetHypsRowSplits( - const std::vector &hyps) { - std::vector row_splits; - row_splits.reserve(hyps.size() + 1); - - row_splits.push_back(0); - int32_t s = 0; - for (const auto &h : hyps) { - s += h.Size(); - row_splits.push_back(s); - } - - return row_splits; -} - std::vector OfflineTransducerModifiedBeamSearchDecoder::Decode( Ort::Value encoder_out, Ort::Value encoder_out_length) { diff --git a/sherpa-onnx/csrc/online-transducer-modified-beam-search-decoder.cc b/sherpa-onnx/csrc/online-transducer-modified-beam-search-decoder.cc index b3ca7c2a..46e5319a 100644 --- a/sherpa-onnx/csrc/online-transducer-modified-beam-search-decoder.cc +++ b/sherpa-onnx/csrc/online-transducer-modified-beam-search-decoder.cc @@ -14,7 +14,7 @@ namespace sherpa_onnx { static void UseCachedDecoderOut( - const std::vector &hyps_num_split, + const std::vector &hyps_row_splits, const std::vector &results, int32_t context_size, Ort::Value *decoder_out) { std::vector shape = @@ -24,7 +24,7 @@ static void UseCachedDecoderOut( int32_t batch_size = static_cast(results.size()); for (int32_t i = 0; i != batch_size; ++i) { - int32_t num_hyps = hyps_num_split[i + 1] - hyps_num_split[i]; + int32_t num_hyps = hyps_row_splits[i + 1] - hyps_row_splits[i]; if (num_hyps > 1 || !results[i].decoder_out) { dst += num_hyps * shape[1]; continue; @@ -86,17 +86,14 @@ void OnlineTransducerModifiedBeamSearchDecoder::Decode( for (int32_t t = 0; t != num_frames; ++t) { // Due to merging paths with identical token sequences, // not all utterances have "num_active_paths" paths. - int32_t hyps_num_acc = 0; - std::vector hyps_num_split; - hyps_num_split.push_back(0); - + auto hyps_row_splits = GetHypsRowSplits(cur); + int32_t num_hyps = + hyps_row_splits.back(); // total num hyps for all utterance prev.clear(); for (auto &hyps : cur) { for (auto &h : hyps) { prev.push_back(std::move(h.second)); - hyps_num_acc++; } - hyps_num_split.push_back(hyps_num_acc); } cur.clear(); cur.reserve(batch_size); @@ -104,30 +101,44 @@ void OnlineTransducerModifiedBeamSearchDecoder::Decode( Ort::Value decoder_input = model_->BuildDecoderInput(prev); Ort::Value decoder_out = model_->RunDecoder(std::move(decoder_input)); if (t == 0) { - UseCachedDecoderOut(hyps_num_split, *result, model_->ContextSize(), + UseCachedDecoderOut(hyps_row_splits, *result, model_->ContextSize(), &decoder_out); } Ort::Value cur_encoder_out = GetEncoderOutFrame(model_->Allocator(), &encoder_out, t); cur_encoder_out = - Repeat(model_->Allocator(), &cur_encoder_out, hyps_num_split); + Repeat(model_->Allocator(), &cur_encoder_out, hyps_row_splits); Ort::Value logit = model_->RunJoiner( std::move(cur_encoder_out), Clone(model_->Allocator(), &decoder_out)); - float *p_logit = logit.GetTensorMutableData(); - for (int32_t b = 0; b < batch_size; ++b) { + float *p_logit = logit.GetTensorMutableData(); + LogSoftmax(p_logit, vocab_size, num_hyps); + + // now p_logit contains log_softmax output, we rename it to p_logprob + // to match what it actually contains + float *p_logprob = p_logit; + + // add log_prob of each hypothesis to p_logprob before taking top_k + for (int32_t i = 0; i != num_hyps; ++i) { + float log_prob = prev[i].log_prob; + for (int32_t k = 0; k != vocab_size; ++k, ++p_logprob) { + *p_logprob += log_prob; + } + } + p_logprob = p_logit; // we changed p_logprob in the above for loop + + for (int32_t b = 0; b != batch_size; ++b) { int32_t frame_offset = (*result)[b].frame_offset; - int32_t start = hyps_num_split[b]; - int32_t end = hyps_num_split[b + 1]; - LogSoftmax(p_logit, vocab_size, (end - start)); + int32_t start = hyps_row_splits[b]; + int32_t end = hyps_row_splits[b + 1]; auto topk = - TopkIndex(p_logit, vocab_size * (end - start), max_active_paths_); + TopkIndex(p_logprob, vocab_size * (end - start), max_active_paths_); Hypotheses hyps; - for (auto i : topk) { - int32_t hyp_index = i / vocab_size + start; - int32_t new_token = i % vocab_size; + for (auto k : topk) { + int32_t hyp_index = k / vocab_size + start; + int32_t new_token = k % vocab_size; Hypothesis new_hyp = prev[hyp_index]; if (new_token != 0) { @@ -137,12 +148,12 @@ void OnlineTransducerModifiedBeamSearchDecoder::Decode( } else { ++new_hyp.num_trailing_blanks; } - new_hyp.log_prob += p_logit[i]; + new_hyp.log_prob = p_logprob[k]; hyps.Add(std::move(new_hyp)); - } + } // for (auto k : topk) cur.push_back(std::move(hyps)); - p_logit += vocab_size * (end - start); - } + p_logprob += (end - start) * vocab_size; + } // for (int32_t b = 0; b != batch_size; ++b) } for (int32_t b = 0; b != batch_size; ++b) {