Export MatchaTTS fa-en model to sherpa-onnx (#1832)

.github/workflows/export-matcha-fa-en.yaml

@@ -0,0 +1,167 @@
+name: export-matcha-fa-en-to-onnx
+
+on:
+  push:
+    branches:
+      - export-matcha-tts-fa-en
+
+  workflow_dispatch:
+
+concurrency:
+  group: export-matcha-fa-en-to-onnx-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  export-kokoro-to-onnx:
+    if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj'
+    name: export matcha fa-en ${{ matrix.version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Python dependencies
+        shell: bash
+        run: |
+          pip install "numpy<=1.26.4" onnx==1.16.0 onnxruntime==1.17.1 soundfile piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html
+
+      - name: Run
+        shell: bash
+        run: |
+          cd scripts/matcha-tts/fa-en
+          ./run.sh
+
+      - name: Collect results ${{ matrix.version }}
+        shell: bash
+        run: |
+          curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2
+          tar xf espeak-ng-data.tar.bz2
+          rm espeak-ng-data.tar.bz2
+
+          src=scripts/matcha-tts/fa-en
+          dst1=matcha-tts-fa_en-male
+          dst2=matcha-tts-fa_en-female
+
+          mkdir $dst1 $dst2
+
+          cp -a espeak-ng-data $dst1/
+          cp -a espeak-ng-data $dst2/
+
+          cp -v $src/male/* $dst1
+          cp -v $src/female/* $dst2
+
+          cp -v $src/README.md $dst1/
+          cp -v $src/README.md $dst2/
+
+          ls -lh $dst1/
+          echo "---"
+          ls -lh $dst2/
+          tar cjfv $dst1.tar.bz2 $dst1
+          tar cjfv $dst2.tar.bz2 $dst2
+
+          ls -lh $dst1.tar.bz2
+          ls -lh $dst2.tar.bz2
+
+      - name: Publish to huggingface male
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "csukuangfj@gmail.com"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_LFS_SKIP_SMUDGE=1
+            export GIT_CLONE_PROTECTION_ACTIVE=false
+
+            git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-male huggingface
+            cd huggingface
+            rm -rf ./*
+            git fetch
+            git pull
+
+            git lfs track "cmn_dict"
+            git lfs track "ru_dict"
+
+            cp -a ../matcha-tts-fa_en-male/* ./
+
+            git lfs track "*.onnx"
+            git add .
+
+            ls -lh
+
+            git status
+
+            git commit -m "add models"
+            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-male main || true
+
+      - name: Publish to huggingface male
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "csukuangfj@gmail.com"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_LFS_SKIP_SMUDGE=1
+            export GIT_CLONE_PROTECTION_ACTIVE=false
+
+            git clone https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-female huggingface
+            cd huggingface
+            rm -rf ./*
+            git fetch
+            git pull
+
+            git lfs track "cmn_dict"
+            git lfs track "ru_dict"
+
+            cp -a ../matcha-tts-fa_en-female/* ./
+
+            git lfs track "*.onnx"
+            git add .
+
+            ls -lh
+
+            git status
+
+            git commit -m "add models"
+            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/matcha-tts-fa_en-female main || true
+
+      - name: Release
+        if: github.repository_owner == 'csukuangfj'
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: ./*.tar.bz2
+          overwrite: true
+          repo_name: k2-fsa/sherpa-onnx
+          repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
+          tag: tts-models
+
+      - name: Release
+        if: github.repository_owner == 'k2-fsa'
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file_glob: true
+          file: ./*.tar.bz2
+          overwrite: true
+          tag: tts-models

scripts/matcha-tts/README.md

@@ -0,0 +1,6 @@
+# Introduction
+
+This folder contains script for adding meta data to tts models
+from https://github.com/shivammehta25/Matcha-TTS
+
+Note: If you use icefall to train a MatchaTTS model, you don't need this folder.

scripts/matcha-tts/fa-en/.gitignore

@@ -0,0 +1 @@
+.add-meta-data.done

scripts/matcha-tts/fa-en/README.md

@@ -0,0 +1,4 @@
+# Introduction
+
+This folder is for
+https://github.com/k2-fsa/sherpa-onnx/issues/1779

scripts/matcha-tts/fa-en/add_meta_data.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+from typing import Any, Dict
+
+import onnx
+
+
+def add_meta_data(filename: str, meta_data: Dict[str, Any]):
+    """Add meta data to an ONNX model. It is changed in-place.
+
+    Args:
+      filename:
+        Filename of the ONNX model to be changed.
+      meta_data:
+        Key-value pairs.
+    """
+    model = onnx.load(filename)
+
+    while len(model.metadata_props):
+        model.metadata_props.pop()
+
+    for key, value in meta_data.items():
+        meta = model.metadata_props.add()
+        meta.key = key
+        meta.value = str(value)
+
+    onnx.save(model, filename)
+
+
+def main():
+    meta_data = {
+        "model_type": "matcha-tts",
+        "language": "Persian+English",
+        "voice": "fa",
+        "has_espeak": 1,
+        "jieba": 0,
+        "n_speakers": 1,
+        "sample_rate": 22050,
+        "version": 1,
+        "pad_id": 0,
+        "use_icefall": 0,
+        "model_author": "Ali Mahmoudi (@mah92)",
+        "maintainer": "k2-fsa",
+        "use_eos_bos": 0,
+        "num_ode_steps": 5,
+        "see_also": "https://github.com/k2-fsa/sherpa-onnx/issues/1779",
+    }
+    add_meta_data("./female/model.onnx", meta_data)
+    add_meta_data("./male/model.onnx", meta_data)
+
+
+if __name__ == "__main__":
+    main()

scripts/matcha-tts/fa-en/run.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+
+set -ex
+mkdir -p female male
+
+if [ ! -f female/model.onnx ]; then
+  curl -SL --output female/model.onnx https://huggingface.co/mah92/Khadijah-FA_EN-Matcha-TTS-Model/resolve/main/matcha-fa-en-khadijah-22050-5.onnx
+fi
+
+if [ ! -f female/tokens.txt ]; then
+  curl -SL --output female/tokens.txt https://huggingface.co/mah92/Khadijah-FA_EN-Matcha-TTS-Model/resolve/main/tokens_sherpa_with_fa.txt
+fi
+
+if [ ! -f male/model.onnx ]; then
+  curl -SL --output male/model.onnx https://huggingface.co/mah92/Musa-FA_EN-Matcha-TTS-Model/resolve/main/matcha-fa-en-musa-22050-5.onnx
+fi
+
+if [ ! -f male/tokens.txt ]; then
+  curl -SL --output male/tokens.txt https://huggingface.co/mah92/Musa-FA_EN-Matcha-TTS-Model/resolve/main/tokens_sherpa_with_fa.txt
+fi
+
+if [ ! -f hifigan_v2.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/hifigan_v2.onnx
+fi
+
+if [ ! -f .add-meta-data.done ]; then
+  python3 ./add_meta_data.py
+  touch .add-meta-data.done
+fi
+
+python3 ./test.py \
+  --am ./female/model.onnx \
+  --vocoder ./hifigan_v2.onnx \
+  --tokens ./female/tokens.txt \
+  --text "This is a test. این یک نمونه ی تست فارسی است." \
+  --out-wav "./female-en-fa.wav"
+
+python3 ./test.py \
+  --am ./male/model.onnx \
+  --vocoder ./hifigan_v2.onnx \
+  --tokens ./male/tokens.txt \
+  --text "This is a test. این یک نمونه ی تست فارسی است." \
+  --out-wav "./male-en-fa.wav"

scripts/matcha-tts/fa-en/test.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+
+"""
+AM
+NodeArg(name='x', type='tensor(int64)', shape=['batch_size', 'time'])
+NodeArg(name='x_lengths', type='tensor(int64)', shape=['batch_size'])
+NodeArg(name='scales', type='tensor(float)', shape=[2])
+-----
+NodeArg(name='mel', type='tensor(float)', shape=['batch_size', 80, 'time'])
+NodeArg(name='mel_lengths', type='tensor(int64)', shape=['batch_size'])
+
+Vocoder
+NodeArg(name='mel', type='tensor(float)', shape=['N', 80, 'L'])
+-----
+NodeArg(name='audio', type='tensor(float)', shape=['N', 'L'])
+"""
+
+import argparse
+
+import numpy as np
+import onnxruntime as ort
+import soundfile as sf
+
+try:
+    from piper_phonemize import phonemize_espeak
+except Exception as ex:
+    raise RuntimeError(
+        f"{ex}\nPlease run\n"
+        "pip install piper_phonemize -f https://k2-fsa.github.io/icefall/piper_phonemize.html"
+    )
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--am", type=str, required=True, help="Path to the acoustic model"
+    )
+
+    parser.add_argument(
+        "--vocoder", type=str, required=True, help="Path to the vocoder"
+    )
+    parser.add_argument(
+        "--tokens", type=str, required=True, help="Path to the tokens.txt"
+    )
+
+    parser.add_argument(
+        "--text", type=str, required=True, help="Path to the text for generation"
+    )
+
+    parser.add_argument(
+        "--out-wav", type=str, required=True, help="Path to save the generated wav"
+    )
+    return parser.parse_args()
+
+
+def load_tokens(filename: str):
+    ans = dict()
+    with open(filename, encoding="utf-8") as f:
+        for line in f:
+            fields = line.strip().split()
+            if len(fields) == 1:
+                ans[" "] = int(fields[0])
+            else:
+                assert len(fields) == 2, (line, fields)
+                ans[fields[0]] = int(fields[1])
+    return ans
+
+
+class OnnxHifiGANModel:
+    def __init__(
+        self,
+        filename: str,
+    ):
+        session_opts = ort.SessionOptions()
+        session_opts.inter_op_num_threads = 1
+        session_opts.intra_op_num_threads = 1
+
+        self.session_opts = session_opts
+        self.model = ort.InferenceSession(
+            filename,
+            sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
+        )
+
+        for i in self.model.get_inputs():
+            print(i)
+
+        print("-----")
+
+        for i in self.model.get_outputs():
+            print(i)
+
+    def __call__(self, x: np.ndarray):
+        assert x.ndim == 3, x.shape
+        assert x.shape[0] == 1, x.shape
+
+        audio = self.model.run(
+            [self.model.get_outputs()[0].name],
+            {
+                self.model.get_inputs()[0].name: x,
+            },
+        )[0]
+        # audio: (batch_size, num_samples)
+
+        return audio
+
+
+class OnnxModel:
+    def __init__(
+        self,
+        filename: str,
+        tokens: str,
+    ):
+        session_opts = ort.SessionOptions()
+        session_opts.inter_op_num_threads = 1
+        session_opts.intra_op_num_threads = 2
+
+        self.session_opts = session_opts
+        self.token2id = load_tokens(tokens)
+        self.model = ort.InferenceSession(
+            filename,
+            sess_options=self.session_opts,
+            providers=["CPUExecutionProvider"],
+        )
+
+        print(f"{self.model.get_modelmeta().custom_metadata_map}")
+        metadata = self.model.get_modelmeta().custom_metadata_map
+        self.sample_rate = int(metadata["sample_rate"])
+
+        for i in self.model.get_inputs():
+            print(i)
+
+        print("-----")
+
+        for i in self.model.get_outputs():
+            print(i)
+
+    def __call__(self, x: np.ndarray):
+        assert x.ndim == 2, x.shape
+        assert x.shape[0] == 1, x.shape
+
+        x_lengths = np.array([x.shape[1]], dtype=np.int64)
+
+        noise_scale = 1.0
+        length_scale = 1.0
+        scales = np.array([noise_scale, length_scale], dtype=np.float32)
+
+        mel = self.model.run(
+            [self.model.get_outputs()[0].name],
+            {
+                self.model.get_inputs()[0].name: x,
+                self.model.get_inputs()[1].name: x_lengths,
+                self.model.get_inputs()[2].name: scales,
+            },
+        )[0]
+        # mel: (batch_size, feat_dim, num_frames)
+
+        return mel
+
+
+def main():
+    args = get_args()
+    print(vars(args))
+    am = OnnxModel(args.am, args.tokens)
+    vocoder = OnnxHifiGANModel(args.vocoder)
+
+    phones = phonemize_espeak(args.text, voice="fa")
+    phones = sum(phones, [])
+    phone_ids = [am.token2id[i] for i in phones]
+
+    padded_phone_ids = [0] * (len(phone_ids) * 2 + 1)
+    padded_phone_ids[1::2] = phone_ids
+
+    tokens = np.array([padded_phone_ids], dtype=np.int64)
+    mel = am(tokens)
+    audio = vocoder(mel)
+
+    sf.write(args.out_wav, audio[0], am.sample_rate, "PCM_16")
+
+
+if __name__ == "__main__":
+    main()