diff --git a/.github/workflows/export-matcha-fa-en.yaml b/.github/workflows/export-matcha-fa-en.yaml new file mode 100644 index 00000000..9d8dbc01 --- /dev/null +++ b/.github/workflows/export-matcha-fa-en.yaml @@ -0,0 +1,167 @@ +name: export-matcha-fa-en-to-onnx + +on: + push: + branches: + - export-matcha-tts-fa-en + + workflow_dispatch: + +concurrency: + group: export-matcha-fa-en-to-onnx-${{ github.ref }} + cancel-in-progress: true + +jobs: + export-kokoro-to-onnx: + if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj' + name: export matcha fa-en ${{ matrix.version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.10"] + + steps: + - uses: actions/checkout@v4 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + shell: bash + run: | + pip install "numpy<=1.26.4" onnx==1.16.0 onnxruntime==1.17.1 soundfile piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html + + - name: Run + shell: bash + run: | + cd scripts/matcha-tts/fa-en + ./run.sh + + - name: Collect results ${{ matrix.version }} + shell: bash + run: | + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2 + tar xf espeak-ng-data.tar.bz2 + rm espeak-ng-data.tar.bz2 + + src=scripts/matcha-tts/fa-en + dst1=matcha-tts-fa_en-male + dst2=matcha-tts-fa_en-female + + mkdir $dst1 $dst2 + + cp -a espeak-ng-data $dst1/ + cp -a espeak-ng-data $dst2/ + + cp -v $src/male/* $dst1 + cp -v $src/female/* $dst2 + + cp -v $src/README.md $dst1/ + cp -v $src/README.md $dst2/ + + ls -lh $dst1/ + echo "---" + ls -lh $dst2/ + tar cjfv $dst1.tar.bz2 $dst1 + tar cjfv $dst2.tar.bz2 $dst2 + + ls -lh $dst1.tar.bz2 + ls -lh $dst2.tar.bz2 + + - name: Publish to huggingface male + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-male huggingface + cd huggingface + rm -rf ./* + git fetch + git pull + + git lfs track "cmn_dict" + git lfs track "ru_dict" + + cp -a ../matcha-tts-fa_en-male/* ./ + + git lfs track "*.onnx" + git add . + + ls -lh + + git status + + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-male main || true + + - name: Publish to huggingface male + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + uses: nick-fields/retry@v3 + with: + max_attempts: 20 + timeout_seconds: 200 + shell: bash + command: | + git config --global user.email "csukuangfj@gmail.com" + git config --global user.name "Fangjun Kuang" + + rm -rf huggingface + export GIT_LFS_SKIP_SMUDGE=1 + export GIT_CLONE_PROTECTION_ACTIVE=false + + git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-female huggingface + cd huggingface + rm -rf ./* + git fetch + git pull + + git lfs track "cmn_dict" + git lfs track "ru_dict" + + cp -a ../matcha-tts-fa_en-female/* ./ + + git lfs track "*.onnx" + git add . + + ls -lh + + git status + + git commit -m "add models" + git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-female main || true + + - name: Release + if: github.repository_owner == 'csukuangfj' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + repo_name: k2-fsa/sherpa-onnx + repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }} + tag: tts-models + + - name: Release + if: github.repository_owner == 'k2-fsa' + uses: svenstaro/upload-release-action@v2 + with: + file_glob: true + file: ./*.tar.bz2 + overwrite: true + tag: tts-models diff --git a/scripts/matcha-tts/README.md b/scripts/matcha-tts/README.md new file mode 100644 index 00000000..f04826e6 --- /dev/null +++ b/scripts/matcha-tts/README.md @@ -0,0 +1,6 @@ +# Introduction + +This folder contains script for adding meta data to tts models +from https://github.com/shivammehta25/Matcha-TTS + +Note: If you use icefall to train a MatchaTTS model, you don't need this folder. diff --git a/scripts/matcha-tts/fa-en/.gitignore b/scripts/matcha-tts/fa-en/.gitignore new file mode 100644 index 00000000..740a502e --- /dev/null +++ b/scripts/matcha-tts/fa-en/.gitignore @@ -0,0 +1 @@ +.add-meta-data.done diff --git a/scripts/matcha-tts/fa-en/README.md b/scripts/matcha-tts/fa-en/README.md new file mode 100644 index 00000000..fe7df537 --- /dev/null +++ b/scripts/matcha-tts/fa-en/README.md @@ -0,0 +1,4 @@ +# Introduction + +This folder is for +https://github.com/k2-fsa/sherpa-onnx/issues/1779 diff --git a/scripts/matcha-tts/fa-en/add_meta_data.py b/scripts/matcha-tts/fa-en/add_meta_data.py new file mode 100755 index 00000000..92eeb5f3 --- /dev/null +++ b/scripts/matcha-tts/fa-en/add_meta_data.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from typing import Any, Dict + +import onnx + + +def add_meta_data(filename: str, meta_data: Dict[str, Any]): + """Add meta data to an ONNX model. It is changed in-place. + + Args: + filename: + Filename of the ONNX model to be changed. + meta_data: + Key-value pairs. + """ + model = onnx.load(filename) + + while len(model.metadata_props): + model.metadata_props.pop() + + for key, value in meta_data.items(): + meta = model.metadata_props.add() + meta.key = key + meta.value = str(value) + + onnx.save(model, filename) + + +def main(): + meta_data = { + "model_type": "matcha-tts", + "language": "Persian+English", + "voice": "fa", + "has_espeak": 1, + "jieba": 0, + "n_speakers": 1, + "sample_rate": 22050, + "version": 1, + "pad_id": 0, + "use_icefall": 0, + "model_author": "Ali Mahmoudi (@mah92)", + "maintainer": "k2-fsa", + "use_eos_bos": 0, + "num_ode_steps": 5, + "see_also": "https://github.com/k2-fsa/sherpa-onnx/issues/1779", + } + add_meta_data("./female/model.onnx", meta_data) + add_meta_data("./male/model.onnx", meta_data) + + +if __name__ == "__main__": + main() diff --git a/scripts/matcha-tts/fa-en/run.sh b/scripts/matcha-tts/fa-en/run.sh new file mode 100755 index 00000000..b445f2b6 --- /dev/null +++ b/scripts/matcha-tts/fa-en/run.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + + +set -ex +mkdir -p female male + +if [ ! -f female/model.onnx ]; then + curl -SL --output female/model.onnx https://huggingface.co/mah92/Khadijah-FA_EN-Matcha-TTS-Model/resolve/main/matcha-fa-en-khadijah-22050-5.onnx +fi + +if [ ! -f female/tokens.txt ]; then + curl -SL --output female/tokens.txt https://huggingface.co/mah92/Khadijah-FA_EN-Matcha-TTS-Model/resolve/main/tokens_sherpa_with_fa.txt +fi + +if [ ! -f male/model.onnx ]; then + curl -SL --output male/model.onnx https://huggingface.co/mah92/Musa-FA_EN-Matcha-TTS-Model/resolve/main/matcha-fa-en-musa-22050-5.onnx +fi + +if [ ! -f male/tokens.txt ]; then + curl -SL --output male/tokens.txt https://huggingface.co/mah92/Musa-FA_EN-Matcha-TTS-Model/resolve/main/tokens_sherpa_with_fa.txt +fi + +if [ ! -f hifigan_v2.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx +fi + +if [ ! -f .add-meta-data.done ]; then + python3 ./add_meta_data.py + touch .add-meta-data.done +fi + +python3 ./test.py \ + --am ./female/model.onnx \ + --vocoder ./hifigan_v2.onnx \ + --tokens ./female/tokens.txt \ + --text "This is a test. این یک نمونه ی تست فارسی است." \ + --out-wav "./female-en-fa.wav" + +python3 ./test.py \ + --am ./male/model.onnx \ + --vocoder ./hifigan_v2.onnx \ + --tokens ./male/tokens.txt \ + --text "This is a test. این یک نمونه ی تست فارسی است." \ + --out-wav "./male-en-fa.wav" diff --git a/scripts/matcha-tts/fa-en/test.py b/scripts/matcha-tts/fa-en/test.py new file mode 100755 index 00000000..1ae54d2c --- /dev/null +++ b/scripts/matcha-tts/fa-en/test.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +""" +AM +NodeArg(name='x', type='tensor(int64)', shape=['batch_size', 'time']) +NodeArg(name='x_lengths', type='tensor(int64)', shape=['batch_size']) +NodeArg(name='scales', type='tensor(float)', shape=[2]) +----- +NodeArg(name='mel', type='tensor(float)', shape=['batch_size', 80, 'time']) +NodeArg(name='mel_lengths', type='tensor(int64)', shape=['batch_size']) + +Vocoder +NodeArg(name='mel', type='tensor(float)', shape=['N', 80, 'L']) +----- +NodeArg(name='audio', type='tensor(float)', shape=['N', 'L']) +""" + +import argparse + +import numpy as np +import onnxruntime as ort +import soundfile as sf + +try: + from piper_phonemize import phonemize_espeak +except Exception as ex: + raise RuntimeError( + f"{ex}\nPlease run\n" + "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html" + ) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--am", type=str, required=True, help="Path to the acoustic model" + ) + + parser.add_argument( + "--vocoder", type=str, required=True, help="Path to the vocoder" + ) + parser.add_argument( + "--tokens", type=str, required=True, help="Path to the tokens.txt" + ) + + parser.add_argument( + "--text", type=str, required=True, help="Path to the text for generation" + ) + + parser.add_argument( + "--out-wav", type=str, required=True, help="Path to save the generated wav" + ) + return parser.parse_args() + + +def load_tokens(filename: str): + ans = dict() + with open(filename, encoding="utf-8") as f: + for line in f: + fields = line.strip().split() + if len(fields) == 1: + ans[" "] = int(fields[0]) + else: + assert len(fields) == 2, (line, fields) + ans[fields[0]] = int(fields[1]) + return ans + + +class OnnxHifiGANModel: + def __init__( + self, + filename: str, + ): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 1 + + self.session_opts = session_opts + self.model = ort.InferenceSession( + filename, + sess_options=self.session_opts, + providers=["CPUExecutionProvider"], + ) + + for i in self.model.get_inputs(): + print(i) + + print("-----") + + for i in self.model.get_outputs(): + print(i) + + def __call__(self, x: np.ndarray): + assert x.ndim == 3, x.shape + assert x.shape[0] == 1, x.shape + + audio = self.model.run( + [self.model.get_outputs()[0].name], + { + self.model.get_inputs()[0].name: x, + }, + )[0] + # audio: (batch_size, num_samples) + + return audio + + +class OnnxModel: + def __init__( + self, + filename: str, + tokens: str, + ): + session_opts = ort.SessionOptions() + session_opts.inter_op_num_threads = 1 + session_opts.intra_op_num_threads = 2 + + self.session_opts = session_opts + self.token2id = load_tokens(tokens) + self.model = ort.InferenceSession( + filename, + sess_options=self.session_opts, + providers=["CPUExecutionProvider"], + ) + + print(f"{self.model.get_modelmeta().custom_metadata_map}") + metadata = self.model.get_modelmeta().custom_metadata_map + self.sample_rate = int(metadata["sample_rate"]) + + for i in self.model.get_inputs(): + print(i) + + print("-----") + + for i in self.model.get_outputs(): + print(i) + + def __call__(self, x: np.ndarray): + assert x.ndim == 2, x.shape + assert x.shape[0] == 1, x.shape + + x_lengths = np.array([x.shape[1]], dtype=np.int64) + + noise_scale = 1.0 + length_scale = 1.0 + scales = np.array([noise_scale, length_scale], dtype=np.float32) + + mel = self.model.run( + [self.model.get_outputs()[0].name], + { + self.model.get_inputs()[0].name: x, + self.model.get_inputs()[1].name: x_lengths, + self.model.get_inputs()[2].name: scales, + }, + )[0] + # mel: (batch_size, feat_dim, num_frames) + + return mel + + +def main(): + args = get_args() + print(vars(args)) + am = OnnxModel(args.am, args.tokens) + vocoder = OnnxHifiGANModel(args.vocoder) + + phones = phonemize_espeak(args.text, voice="fa") + phones = sum(phones, []) + phone_ids = [am.token2id[i] for i in phones] + + padded_phone_ids = [0] * (len(phone_ids) * 2 + 1) + padded_phone_ids[1::2] = phone_ids + + tokens = np.array([padded_phone_ids], dtype=np.int64) + mel = am(tokens) + audio = vocoder(mel) + + sf.write(args.out_wav, audio[0], am.sample_rate, "PCM_16") + + +if __name__ == "__main__": + main()