EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Support Matcha-TTS models using espeak-ng (#1672)

Browse Source
This commit is contained in:
Fangjun Kuang
2025-01-02 13:46:43 +08:00
committed by GitHub
parent 3422b9388d
commit f457baea42
10 changed files with 290 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
sherpa-onnx/csrc/macros.h
View File

@@ -49,19 +49,21 @@
} while (0)
#endif
#define SHERPA_ONNX_EXIT(code) exit(code)
// Read an integer
#define SHERPA_ONNX_READ_META_DATA(dst, src_key) \
do { \
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
\
dst = atoi(value.c_str()); \
if (dst < 0) { \
SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -74,7 +76,7 @@
dst = atoi(value.c_str()); \
if (dst < 0) { \
SHERPA_ONNX_LOGE("Invalid value %d for '%s'", dst, src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} \
} while (0)
@@ -85,13 +87,13 @@
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
\
bool ret = SplitStringToIntegers(value.c_str(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -101,13 +103,13 @@
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("%s does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
\
bool ret = SplitStringToFloats(value.c_str(), ",", true, &dst); \
if (!ret) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'", value.c_str(), src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -117,14 +119,14 @@
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
SplitStringToVector(value.c_str(), ",", false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \
value.c_str(), src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -134,14 +136,14 @@
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
SplitStringToVector(value.c_str(), sep, false, &dst); \
\
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value '%s' for '%s'. Empty vector!", \
value.c_str(), src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -151,13 +153,13 @@
auto value = LookupCustomModelMetaData(meta_data, src_key, allocator); \
if (value.empty()) { \
SHERPA_ONNX_LOGE("'%s' does not exist in the metadata", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
\
dst = std::move(value); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} while (0)
@@ -178,11 +180,9 @@
dst = std::move(value); \
if (dst.empty()) { \
SHERPA_ONNX_LOGE("Invalid value for '%s'\n", src_key); \
exit(-1); \
SHERPA_ONNX_EXIT(-1); \
} \
} \
} while (0)
#define SHERPA_ONNX_EXIT(code) exit(code)
#endif // SHERPA_ONNX_CSRC_MACROS_H_

41
sherpa-onnx/csrc/offline-tts-matcha-impl.h
View File

@@ -321,12 +321,45 @@ class OfflineTtsMatchaImpl : public OfflineTtsImpl {
private:
template <typename Manager>
void InitFrontend(Manager *mgr) {}
void InitFrontend(Manager *mgr) {
// for piper phonemizer
// we require that you copy espeak_ng_data
// from assets to disk
//
// for jieba
// we require that you copy tokens.txt, lexicon.txt and dict
// from assets to disk
const auto &meta_data = model_->GetMetaData();
if (meta_data.jieba && !meta_data.has_espeak) {
frontend_ = std::make_unique<JiebaLexicon>(
config_.model.matcha.lexicon, config_.model.matcha.tokens,
config_.model.matcha.dict_dir, config_.model.debug);
} else if (meta_data.has_espeak && !meta_data.jieba) {
frontend_ = std::make_unique<PiperPhonemizeLexicon>(
mgr, config_.model.matcha.tokens, config_.model.matcha.data_dir,
meta_data);
} else {
SHERPA_ONNX_LOGE("jieba + espeaker-ng is not supported yet");
SHERPA_ONNX_EXIT(-1);
}
}
void InitFrontend() {
frontend_ = std::make_unique<JiebaLexicon>(
config_.model.matcha.lexicon, config_.model.matcha.tokens,
config_.model.matcha.dict_dir, config_.model.debug);
const auto &meta_data = model_->GetMetaData();
if (meta_data.jieba && !meta_data.has_espeak) {
frontend_ = std::make_unique<JiebaLexicon>(
config_.model.matcha.lexicon, config_.model.matcha.tokens,
config_.model.matcha.dict_dir, config_.model.debug);
} else if (meta_data.has_espeak && !meta_data.jieba) {
frontend_ = std::make_unique<PiperPhonemizeLexicon>(
config_.model.matcha.tokens, config_.model.matcha.data_dir,
meta_data);
} else {
SHERPA_ONNX_LOGE("jieba + espeaker-ng is not supported yet");
SHERPA_ONNX_EXIT(-1);
}
}
GeneratedAudio Process(const std::vector<std::vector<int64_t>> &tokens,

2
sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h
View File

@@ -18,7 +18,7 @@ struct OfflineTtsMatchaModelMetaData {
int32_t num_speakers = 0;
int32_t version = 1;
int32_t jieba = 0;
int32_t espeak = 0;
int32_t has_espeak = 0;
int32_t use_eos_bos = 0;
int32_t pad_id = 0;
};

2
sherpa-onnx/csrc/offline-tts-matcha-model.cc
View File

@@ -142,7 +142,7 @@ class OfflineTtsMatchaModel::Impl {
SHERPA_ONNX_READ_META_DATA_WITH_DEFAULT(meta_data_.version, "version", 1);
SHERPA_ONNX_READ_META_DATA(meta_data_.num_speakers, "n_speakers");
SHERPA_ONNX_READ_META_DATA(meta_data_.jieba, "jieba");
SHERPA_ONNX_READ_META_DATA(meta_data_.espeak, "has_espeak");
SHERPA_ONNX_READ_META_DATA(meta_data_.has_espeak, "has_espeak");
SHERPA_ONNX_READ_META_DATA(meta_data_.use_eos_bos, "use_eos_bos");
SHERPA_ONNX_READ_META_DATA(meta_data_.pad_id, "pad_id");
}

151
sherpa-onnx/csrc/piper-phonemize-lexicon.cc
View File

@@ -32,6 +32,18 @@
namespace sherpa_onnx {
static void CallPhonemizeEspeak(
const std::string &text,
piper::eSpeakPhonemeConfig &config, // NOLINT
std::vector<std::vector<piper::Phoneme>> *phonemes) {
static std::mutex espeak_mutex;
std::lock_guard<std::mutex> lock(espeak_mutex);
// keep multi threads from calling into piper::phonemize_eSpeak
piper::phonemize_eSpeak(text, config, *phonemes);
}
static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> conv;
std::unordered_map<char32_t, int32_t> token2id;
@@ -87,7 +99,7 @@ static std::unordered_map<char32_t, int32_t> ReadTokens(std::istream &is) {
// see the function "phonemes_to_ids" from
// https://github.com/rhasspy/piper/blob/master/notebooks/piper_inference_(ONNX).ipynb
static std::vector<int64_t> PiperPhonemesToIds(
static std::vector<int64_t> PiperPhonemesToIdsVits(
const std::unordered_map<char32_t, int32_t> &token2id,
const std::vector<piper::Phoneme> &phonemes) {
// see
@@ -114,17 +126,46 @@ static std::vector<int64_t> PiperPhonemesToIds(
return ans;
}
static std::vector<int64_t> PiperPhonemesToIdsMatcha(
const std::unordered_map<char32_t, int32_t> &token2id,
const std::vector<piper::Phoneme> &phonemes, bool use_eos_bos) {
std::vector<int64_t> ans;
ans.reserve(phonemes.size());
int32_t bos = token2id.at(U'^');
int32_t eos = token2id.at(U'$');
if (use_eos_bos) {
ans.push_back(bos);
}
for (auto p : phonemes) {
if (token2id.count(p)) {
ans.push_back(token2id.at(p));
} else {
SHERPA_ONNX_LOGE("Skip unknown phonemes. Unicode codepoint: \\U+%04x.",
static_cast<uint32_t>(p));
}
}
if (use_eos_bos) {
ans.push_back(eos);
}
return ans;
}
static std::vector<int64_t> CoquiPhonemesToIds(
const std::unordered_map<char32_t, int32_t> &token2id,
const std::vector<piper::Phoneme> &phonemes,
const OfflineTtsVitsModelMetaData &meta_data) {
const OfflineTtsVitsModelMetaData &vits_meta_data) {
// see
// https://github.com/coqui-ai/TTS/blob/dev/TTS/tts/utils/text/tokenizer.py#L87
int32_t use_eos_bos = meta_data.use_eos_bos;
int32_t bos_id = meta_data.bos_id;
int32_t eos_id = meta_data.eos_id;
int32_t blank_id = meta_data.blank_id;
int32_t add_blank = meta_data.add_blank;
int32_t use_eos_bos = vits_meta_data.use_eos_bos;
int32_t bos_id = vits_meta_data.bos_id;
int32_t eos_id = vits_meta_data.eos_id;
int32_t blank_id = vits_meta_data.blank_id;
int32_t add_blank = vits_meta_data.add_blank;
int32_t comma_id = token2id.at(',');
std::vector<int64_t> ans;
@@ -189,8 +230,8 @@ static void InitEspeak(const std::string &data_dir) {
PiperPhonemizeLexicon::PiperPhonemizeLexicon(
const std::string &tokens, const std::string &data_dir,
const OfflineTtsVitsModelMetaData &meta_data)
: meta_data_(meta_data) {
const OfflineTtsVitsModelMetaData &vits_meta_data)
: vits_meta_data_(vits_meta_data) {
{
std::ifstream is(tokens);
token2id_ = ReadTokens(is);
@@ -202,8 +243,37 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon(
template <typename Manager>
PiperPhonemizeLexicon::PiperPhonemizeLexicon(
Manager *mgr, const std::string &tokens, const std::string &data_dir,
const OfflineTtsVitsModelMetaData &meta_data)
: meta_data_(meta_data) {
const OfflineTtsVitsModelMetaData &vits_meta_data)
: vits_meta_data_(vits_meta_data) {
{
auto buf = ReadFile(mgr, tokens);
std::istrstream is(buf.data(), buf.size());
token2id_ = ReadTokens(is);
}
// We should copy the directory of espeak-ng-data from the asset to
// some internal or external storage and then pass the directory to
// data_dir.
InitEspeak(data_dir);
}
PiperPhonemizeLexicon::PiperPhonemizeLexicon(
const std::string &tokens, const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data)
: matcha_meta_data_(matcha_meta_data), is_matcha_(true) {
{
std::ifstream is(tokens);
token2id_ = ReadTokens(is);
}
InitEspeak(data_dir);
}
template <typename Manager>
PiperPhonemizeLexicon::PiperPhonemizeLexicon(
Manager *mgr, const std::string &tokens, const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data)
: matcha_meta_data_(matcha_meta_data), is_matcha_(true) {
{
auto buf = ReadFile(mgr, tokens);
std::istrstream is(buf.data(), buf.size());
@@ -218,6 +288,15 @@ PiperPhonemizeLexicon::PiperPhonemizeLexicon(
std::vector<TokenIDs> PiperPhonemizeLexicon::ConvertTextToTokenIds(
const std::string &text, const std::string &voice /*= ""*/) const {
if (is_matcha_) {
return ConvertTextToTokenIdsMatcha(text, voice);
} else {
return ConvertTextToTokenIdsVits(text, voice);
}
}
std::vector<TokenIDs> PiperPhonemizeLexicon::ConvertTextToTokenIdsMatcha(
const std::string &text, const std::string &voice /*= ""*/) const {
piper::eSpeakPhonemeConfig config;
// ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices
@@ -226,26 +305,45 @@ std::vector<TokenIDs> PiperPhonemizeLexicon::ConvertTextToTokenIds(
std::vector<std::vector<piper::Phoneme>> phonemes;
static std::mutex espeak_mutex;
{
std::lock_guard<std::mutex> lock(espeak_mutex);
// keep multi threads from calling into piper::phonemize_eSpeak
piper::phonemize_eSpeak(text, config, phonemes);
}
CallPhonemizeEspeak(text, config, &phonemes);
std::vector<TokenIDs> ans;
std::vector<int64_t> phoneme_ids;
if (meta_data_.is_piper || meta_data_.is_icefall) {
for (const auto &p : phonemes) {
phoneme_ids =
PiperPhonemesToIdsMatcha(token2id_, p, matcha_meta_data_.use_eos_bos);
ans.emplace_back(std::move(phoneme_ids));
}
return ans;
}
std::vector<TokenIDs> PiperPhonemizeLexicon::ConvertTextToTokenIdsVits(
const std::string &text, const std::string &voice /*= ""*/) const {
piper::eSpeakPhonemeConfig config;
// ./bin/espeak-ng-bin --path ./install/share/espeak-ng-data/ --voices
// to list available voices
config.voice = voice; // e.g., voice is en-us
std::vector<std::vector<piper::Phoneme>> phonemes;
CallPhonemizeEspeak(text, config, &phonemes);
std::vector<TokenIDs> ans;
std::vector<int64_t> phoneme_ids;
if (vits_meta_data_.is_piper || vits_meta_data_.is_icefall) {
for (const auto &p : phonemes) {
phoneme_ids = PiperPhonemesToIds(token2id_, p);
phoneme_ids = PiperPhonemesToIdsVits(token2id_, p);
ans.emplace_back(std::move(phoneme_ids));
}
} else if (meta_data_.is_coqui) {
} else if (vits_meta_data_.is_coqui) {
for (const auto &p : phonemes) {
phoneme_ids = CoquiPhonemesToIds(token2id_, p, meta_data_);
phoneme_ids = CoquiPhonemesToIds(token2id_, p, vits_meta_data_);
ans.emplace_back(std::move(phoneme_ids));
}
@@ -260,13 +358,18 @@ std::vector<TokenIDs> PiperPhonemizeLexicon::ConvertTextToTokenIds(
#if __ANDROID_API__ >= 9
template PiperPhonemizeLexicon::PiperPhonemizeLexicon(
AAssetManager *mgr, const std::string &tokens, const std::string &data_dir,
const OfflineTtsVitsModelMetaData &meta_data);
const OfflineTtsVitsModelMetaData &vits_meta_data);
template PiperPhonemizeLexicon::PiperPhonemizeLexicon(
AAssetManager *mgr, const std::string &tokens, const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data);
#endif
#if __OHOS__
template PiperPhonemizeLexicon::PiperPhonemizeLexicon(
NativeResourceManager *mgr, const std::string &tokens,
const std::string &data_dir, const OfflineTtsVitsModelMetaData &meta_data);
const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data);
#endif
} // namespace sherpa_onnx

24
sherpa-onnx/csrc/piper-phonemize-lexicon.h
View File

@@ -10,6 +10,7 @@
#include <vector>
#include "sherpa-onnx/csrc/offline-tts-frontend.h"
#include "sherpa-onnx/csrc/offline-tts-matcha-model-metadata.h"
#include "sherpa-onnx/csrc/offline-tts-vits-model-metadata.h"
namespace sherpa_onnx {
@@ -17,20 +18,37 @@ namespace sherpa_onnx {
class PiperPhonemizeLexicon : public OfflineTtsFrontend {
public:
PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir,
const OfflineTtsVitsModelMetaData &meta_data);
const OfflineTtsVitsModelMetaData &vits_meta_data);
PiperPhonemizeLexicon(const std::string &tokens, const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data);
template <typename Manager>
PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens,
const std::string &data_dir,
const OfflineTtsVitsModelMetaData &meta_data);
const OfflineTtsVitsModelMetaData &vits_meta_data);
template <typename Manager>
PiperPhonemizeLexicon(Manager *mgr, const std::string &tokens,
const std::string &data_dir,
const OfflineTtsMatchaModelMetaData &matcha_meta_data);
std::vector<TokenIDs> ConvertTextToTokenIds(
const std::string &text, const std::string &voice = "") const override;
private:
std::vector<TokenIDs> ConvertTextToTokenIdsVits(
const std::string &text, const std::string &voice = "") const;
std::vector<TokenIDs> ConvertTextToTokenIdsMatcha(
const std::string &text, const std::string &voice = "") const;
private:
// map unicode codepoint to an integer ID
std::unordered_map<char32_t, int32_t> token2id_;
OfflineTtsVitsModelMetaData meta_data_;
OfflineTtsVitsModelMetaData vits_meta_data_;
OfflineTtsMatchaModelMetaData matcha_meta_data_;
bool is_matcha_ = false;
};
} // namespace sherpa_onnx