// sherpa-onnx/csrc/offline-recognizer-ctc-impl.h // // Copyright (c) 2022-2023 Xiaomi Corporation #ifndef SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_CTC_IMPL_H_ #define SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_CTC_IMPL_H_ #include #include #include #include #if __ANDROID_API__ >= 9 #include "android/asset_manager.h" #include "android/asset_manager_jni.h" #endif #include "sherpa-onnx/csrc/offline-ctc-decoder.h" #include "sherpa-onnx/csrc/offline-ctc-fst-decoder.h" #include "sherpa-onnx/csrc/offline-ctc-greedy-search-decoder.h" #include "sherpa-onnx/csrc/offline-ctc-model.h" #include "sherpa-onnx/csrc/offline-recognizer-impl.h" #include "sherpa-onnx/csrc/pad-sequence.h" #include "sherpa-onnx/csrc/symbol-table.h" namespace sherpa_onnx { static OfflineRecognitionResult Convert(const OfflineCtcDecoderResult &src, const SymbolTable &sym_table, int32_t frame_shift_ms, int32_t subsampling_factor) { OfflineRecognitionResult r; r.tokens.reserve(src.tokens.size()); r.timestamps.reserve(src.timestamps.size()); std::string text; for (int32_t i = 0; i != src.tokens.size(); ++i) { if (sym_table.contains("SIL") && src.tokens[i] == sym_table["SIL"]) { // tdnn models from yesno have a SIL token, we should remove it. continue; } auto sym = sym_table[src.tokens[i]]; text.append(sym); r.tokens.push_back(std::move(sym)); } r.text = std::move(text); float frame_shift_s = frame_shift_ms / 1000. * subsampling_factor; for (auto t : src.timestamps) { float time = frame_shift_s * t; r.timestamps.push_back(time); } return r; } class OfflineRecognizerCtcImpl : public OfflineRecognizerImpl { public: explicit OfflineRecognizerCtcImpl(const OfflineRecognizerConfig &config) : config_(config), symbol_table_(config_.model_config.tokens), model_(OfflineCtcModel::Create(config_.model_config)) { Init(); } #if __ANDROID_API__ >= 9 OfflineRecognizerCtcImpl(AAssetManager *mgr, const OfflineRecognizerConfig &config) : config_(config), symbol_table_(mgr, config_.model_config.tokens), model_(OfflineCtcModel::Create(mgr, config_.model_config)) { Init(); } #endif void Init() { config_.feat_config.nemo_normalize_type = model_->FeatureNormalizationMethod(); if (!config_.ctc_fst_decoder_config.graph.empty()) { // TODO(fangjun): Support android to read the graph from // asset_manager decoder_ = std::make_unique( config_.ctc_fst_decoder_config); } else if (config_.decoding_method == "greedy_search") { if (!symbol_table_.contains("") && !symbol_table_.contains("")) { SHERPA_ONNX_LOGE( "We expect that tokens.txt contains " "the symbol or and its ID."); exit(-1); } int32_t blank_id = 0; if (symbol_table_.contains("")) { blank_id = symbol_table_[""]; } else if (symbol_table_.contains("")) { // for tdnn models of the yesno recipe from icefall blank_id = symbol_table_[""]; } decoder_ = std::make_unique(blank_id); } else { SHERPA_ONNX_LOGE("Only greedy_search is supported at present. Given %s", config_.decoding_method.c_str()); exit(-1); } } std::unique_ptr CreateStream() const override { return std::make_unique(config_.feat_config); } void DecodeStreams(OfflineStream **ss, int32_t n) const override { auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeDefault); int32_t feat_dim = config_.feat_config.feature_dim; std::vector features; features.reserve(n); std::vector> features_vec(n); std::vector features_length_vec(n); for (int32_t i = 0; i != n; ++i) { std::vector f = ss[i]->GetFrames(); int32_t num_frames = f.size() / feat_dim; features_vec[i] = std::move(f); features_length_vec[i] = num_frames; std::array shape = {num_frames, feat_dim}; Ort::Value x = Ort::Value::CreateTensor( memory_info, features_vec[i].data(), features_vec[i].size(), shape.data(), shape.size()); features.push_back(std::move(x)); } // for (int32_t i = 0; i != n; ++i) std::vector features_pointer(n); for (int32_t i = 0; i != n; ++i) { features_pointer[i] = &features[i]; } std::array features_length_shape = {n}; Ort::Value x_length = Ort::Value::CreateTensor( memory_info, features_length_vec.data(), n, features_length_shape.data(), features_length_shape.size()); Ort::Value x = PadSequence(model_->Allocator(), features_pointer, -23.025850929940457f); auto t = model_->Forward(std::move(x), std::move(x_length)); auto results = decoder_->Decode(std::move(t[0]), std::move(t[1])); int32_t frame_shift_ms = 10; for (int32_t i = 0; i != n; ++i) { auto r = Convert(results[i], symbol_table_, frame_shift_ms, model_->SubsamplingFactor()); ss[i]->SetResult(r); } } private: OfflineRecognizerConfig config_; SymbolTable symbol_table_; std::unique_ptr model_; std::unique_ptr decoder_; }; } // namespace sherpa_onnx #endif // SHERPA_ONNX_CSRC_OFFLINE_RECOGNIZER_CTC_IMPL_H_