EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add nuget package for Windows x86 (#683)

Browse Source
This commit is contained in:
Fangjun Kuang
2024-03-21 14:57:01 +08:00
committed by GitHub
parent acf0975153
commit c8770aec20
5 changed files with 82 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
sherpa-onnx/csrc/lexicon.cc
View File

@@ -145,7 +145,9 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIds(
}
std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
const std::string &text) const {
const std::string &_text) const {
std::string text(_text);
ToLowerCase(&text);
std::vector<std::string> words;
if (pattern_) {
// Handle polyphones
@@ -206,6 +208,11 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
eos = token2id_.at("eos");
}
int32_t pad = -1;
if (token2id_.count("#0")) {
pad = token2id_.at("#0");
}
if (sil != -1) {
this_sentence.push_back(sil);
}
@@ -219,6 +226,8 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
if (punctuations_.count(w)) {
if (token2id_.count(w)) {
this_sentence.push_back(token2id_.at(w));
} else if (pad != -1) {
this_sentence.push_back(pad);
} else if (sil != -1) {
this_sentence.push_back(sil);
}