Add C++ and Python API for Kokoro 1.0 multilingual TTS model (#1795)

2025-02-06 22:57:13 +08:00
parent 08cefe8488
commit c84a833863
20 changed files with 819 additions and 39 deletions
--- a/python-api-examples/offline-tts-play.py
+++ b/python-api-examples/offline-tts-play.py
@@ -11,7 +11,7 @@ while the model is still generating.

 Usage:

-Example (1/6)
+Example (1/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2
 tar xf vits-piper-en_US-amy-low.tar.bz2
@@ -23,7 +23,7 @@ python3 ./python-api-examples/offline-tts-play.py \
 --output-filename=./generated.wav \
 "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."

-Example (2/6)
+Example (2/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2
 tar xvf vits-zh-aishell3.tar.bz2
@@ -37,7 +37,7 @@ python3 ./python-api-examples/offline-tts-play.py \
 --output-filename=./liubei-21.wav \
 "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334"

-Example (3/6)
+Example (3/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2
 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2
@@ -53,7 +53,7 @@ python3 ./python-api-examples/offline-tts-play.py \
 --output-filename=./test-2.wav \
 "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。"

-Example (4/6)
+Example (4/7)

 curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2
 tar xvf matcha-icefall-zh-baker.tar.bz2
@@ -71,7 +71,7 @@ python3 ./python-api-examples/offline-tts-play.py \
 --output-filename=./test-matcha.wav \
 "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。"

-Example (5/6)
+Example (5/7)

 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2
 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2
@@ -88,7 +88,9 @@ python3 ./python-api-examples/offline-tts-play.py \
  --num-threads=2 \
 "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."

-Example (6/6)
+Example (6/7)
+
+(This version of kokoro supports only English)

 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2
 tar xf kokoro-en-v0_19.tar.bz2
@@ -105,6 +107,27 @@ python3 ./python-api-examples/offline-tts.py \
  --output-filename="./kokoro-10.wav" \
  "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be  a statesman, a businessman, an official, or a scholar."

+Example (7/7)
+
+(This version of kokoro supports English, Chinese, etc.)
+
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
+tar xf kokoro-multi-lang-v1_0.tar.bz2
+rm kokoro-multi-lang-v1_0.tar.bz2
+
+python3 ./python-api-examples/offline-tts-play.py \
+  --debug=1 \
+  --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \
+  --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \
+  --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \
+  --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \
+  --kokoro-dict-dir=./kokoro-multi-lang-v1_0/dict \
+  --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \
+  --num-threads=2 \
+  --sid=18 \
+  --output-filename="./kokoro-18-zh-en.wav" \
+  "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？"
+
 You can find more models at
 https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models

@@ -247,6 +270,20 @@ def add_kokoro_args(parser):
        help="Path to the dict directory of espeak-ng.",
    )

+    parser.add_argument(
+        "--kokoro-dict-dir",
+        type=str,
+        default="",
+        help="Path to the dict directory for models using jieba. Needed only by multilingual kokoro",
+    )
+
+    parser.add_argument(
+        "--kokoro-lexicon",
+        type=str,
+        default="",
+        help="Path to lexicon.txt for kokoro. Needed only by multilingual kokoro",
+    )
+

 def get_args():
    parser = argparse.ArgumentParser(
@@ -459,6 +496,8 @@ def main():
                voices=args.kokoro_voices,
                tokens=args.kokoro_tokens,
                data_dir=args.kokoro_data_dir,
+                dict_dir=args.kokoro_dict_dir,
+                lexicon=args.kokoro_lexicon,
            ),
            provider=args.provider,
            debug=args.debug,
--- a/python-api-examples/offline-tts.py
+++ b/python-api-examples/offline-tts.py
@@ -12,7 +12,7 @@ generated audio.

 Usage:

-Example (1/6)
+Example (1/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2
 tar xf vits-piper-en_US-amy-low.tar.bz2
@@ -24,7 +24,7 @@ python3 ./python-api-examples/offline-tts.py \
 --output-filename=./generated.wav \
 "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."

-Example (2/6)
+Example (2/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2
 tar xvf vits-icefall-zh-aishell3.tar.bz2
@@ -38,7 +38,7 @@ python3 ./python-api-examples/offline-tts.py \
 --output-filename=./liubei-21.wav \
 "勿以恶小而为之，勿以善小而不为。惟贤惟德，能服于人。122334"

-Example (3/6)
+Example (3/7)

 wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2
 tar xvf sherpa-onnx-vits-zh-ll.tar.bz2
@@ -54,7 +54,7 @@ python3 ./python-api-examples/offline-tts.py \
 --output-filename=./test-2.wav \
 "当夜幕降临，星光点点，伴随着微风拂面，我在静谧中感受着时光的流转，思念如涟漪荡漾，梦境如画卷展开，我与自然融为一体，沉静在这片宁静的美丽之中，感受着生命的奇迹与温柔。2024年5月11号，拨打110或者18920240511。123456块钱。"

-Example (4/6)
+Example (4/7)

 curl -O -SL https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-zh-baker.tar.bz2
 tar xvf matcha-icefall-zh-baker.tar.bz2
@@ -72,7 +72,7 @@ python3 ./python-api-examples/offline-tts.py \
 --output-filename=./test-matcha.wav \
 "某某银行的副行长和一些行政领导表示，他们去过长江和长白山; 经济不断增长。2024年12月31号，拨打110或者18920240511。123456块钱。"

-Example (5/6)
+Example (5/7)

 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/matcha-icefall-en_US-ljspeech.tar.bz2
 tar xvf matcha-icefall-en_US-ljspeech.tar.bz2
@@ -89,7 +89,9 @@ python3 ./python-api-examples/offline-tts.py \
  --num-threads=2 \
 "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be: a statesman, a businessman, an official, or a scholar."

-Example (6/6)
+Example (6/7)
+
+(This version of kokoro supports only English)

 curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-en-v0_19.tar.bz2
 tar xf kokoro-en-v0_19.tar.bz2
@@ -106,6 +108,27 @@ python3 ./python-api-examples/offline-tts.py \
  --output-filename="./kokoro-10.wav" \
  "Today as always, men fall into two groups: slaves and free men. Whoever does not have two-thirds of his day for himself, is a slave, whatever he may be  a statesman, a businessman, an official, or a scholar."

+Example (7/7)
+
+(This version of kokoro supports English, Chinese, etc.)
+
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/kokoro-multi-lang-v1_0.tar.bz2
+tar xf kokoro-multi-lang-v1_0.tar.bz2
+rm kokoro-multi-lang-v1_0.tar.bz2
+
+python3 ./python-api-examples/offline-tts.py \
+  --debug=1 \
+  --kokoro-model=./kokoro-multi-lang-v1_0/model.onnx \
+  --kokoro-voices=./kokoro-multi-lang-v1_0/voices.bin \
+  --kokoro-tokens=./kokoro-multi-lang-v1_0/tokens.txt \
+  --kokoro-data-dir=./kokoro-multi-lang-v1_0/espeak-ng-data \
+  --kokoro-dict-dir=./kokoro-multi-lang-v1_0/dict \
+  --kokoro-lexicon=./kokoro-multi-lang-v1_0/lexicon-us-en.txt,./kokoro-multi-lang-v1_0/lexicon-zh.txt \
+  --num-threads=2 \
+  --sid=18 \
+  --output-filename="./kokoro-18-zh-en.wav" \
+  "中英文语音合成测试。This is generated by next generation Kaldi using Kokoro without Misaki. 你觉得中英文说的如何呢？"
+
 You can find more models at
 https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models

@@ -234,6 +257,20 @@ def add_kokoro_args(parser):
        help="Path to the dict directory of espeak-ng.",
    )

+    parser.add_argument(
+        "--kokoro-dict-dir",
+        type=str,
+        default="",
+        help="Path to the dict directory for models using jieba. Needed only by multilingual kokoro",
+    )
+
+    parser.add_argument(
+        "--kokoro-lexicon",
+        type=str,
+        default="",
+        help="Path to lexicon.txt for kokoro. Needed only by multilingual kokoro",
+    )
+

 def get_args():
    parser = argparse.ArgumentParser(
@@ -342,6 +379,8 @@ def main():
                voices=args.kokoro_voices,
                tokens=args.kokoro_tokens,
                data_dir=args.kokoro_data_dir,
+                dict_dir=args.kokoro_dict_dir,
+                lexicon=args.kokoro_lexicon,
            ),
            provider=args.provider,
            debug=args.debug,