EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Support Chinese vits models (#368)

Browse Source
This commit is contained in:
Fangjun Kuang
2023-10-18 10:19:10 +08:00
committed by GitHub
parent 9efe69720d
commit 1ee79e3ff5
16 changed files with 326 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
sherpa-onnx/csrc/lexicon.h
View File

@@ -13,18 +13,40 @@
namespace sherpa_onnx {
// TODO(fangjun): Refactor it to an abstract class
class Lexicon {
public:
Lexicon(const std::string &lexicon, const std::string &tokens,
const std::string &punctuations);
const std::string &punctuations, const std::string &language);
std::vector<int64_t> ConvertTextToTokenIds(const std::string &text) const;
private:
std::vector<int64_t> ConvertTextToTokenIdsEnglish(
const std::string &text) const;
std::vector<int64_t> ConvertTextToTokenIdsChinese(
const std::string &text) const;
void InitLanguage(const std::string &lang);
void InitTokens(const std::string &tokens);
void InitLexicon(const std::string &lexicon);
void InitPunctuations(const std::string &punctuations);
private:
enum class Language {
kEnglish,
kChinese,
kUnknown,
};
private:
std::unordered_map<std::string, std::vector<int32_t>> word2ids_;
std::unordered_set<std::string> punctuations_;
std::unordered_map<std::string, int32_t> token2id_;
int32_t blank_; // ID for the blank token
int32_t blank_ = -1; // ID for the blank token
Language language_;
//
};
} // namespace sherpa_onnx