Add TTS demo for C# API (#557)

.github/workflows/test-dot-net-nuget.yaml

@@ -55,3 +55,19 @@ jobs:
           ./run-zipformer.sh
           ./run-whisper.sh
           ./run-tdnn-yesno.sh
+
+          cd ../offline-tts
+          ./run-aishell3.sh
+          ./run-piper.sh
+          ls -lh
+
+          cd ../..
+
+          mkdir tts
+
+          cp dotnet-examples/offline-tts/*.wav ./tts
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: dot-net-tts-generated-test-files-${{ matrix.os }}
+          path: tts

.github/workflows/test-dot-net.yaml

@@ -131,6 +131,7 @@ jobs:
       - name: Copy files
         shell: bash
         run: |
+          cp -v scripts/dotnet/examples/offline-tts.csproj dotnet-examples/offline-tts/
           cp -v scripts/dotnet/examples/offline-decode-files.csproj dotnet-examples/offline-decode-files/
           cp -v scripts/dotnet/examples/online-decode-files.csproj dotnet-examples/online-decode-files/
           cp -v scripts/dotnet/examples/speech-recognition-from-microphone.csproj dotnet-examples/speech-recognition-from-microphone/
@@ -153,3 +154,19 @@ jobs:
           ./run-zipformer.sh
           ./run-whisper.sh
           ./run-tdnn-yesno.sh
+
+          cd ../offline-tts
+          ./run-aishell3.sh
+          ./run-piper.sh
+          ls -lh
+
+          cd ../..
+
+          mkdir tts
+
+          cp dotnet-examples/offline-tts/*.wav ./tts
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: dot-net-tts-generated-test-files-${{ matrix.os }}
+          path: tts

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-onnx)
 
-set(SHERPA_ONNX_VERSION "1.9.9")
+set(SHERPA_ONNX_VERSION "1.9.10")
 
 # Disable warning about
 #

dotnet-examples/.notes

@@ -0,0 +1,9 @@
+# How to create a new project in this folder
+
+```bash
+mkdir offline-tts
+cd offline-tts
+dotnet new console
+cd ..
+dotnet sln ./sherpa-onnx.sln add ./offline-tts
+```

dotnet-examples/offline-tts/Program.cs

@@ -0,0 +1,147 @@
+// Copyright (c)  2024  Xiaomi Corporation
+//
+// This file shows how to use a non-streaming TTS model for text-to-speech
+// Please refer to
+// https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
+// and
+// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+// to download pre-trained models
+using CommandLine.Text;
+using CommandLine;
+using SherpaOnnx;
+using System.Collections.Generic;
+using System;
+
+class OfflineTtsDemo
+{
+  class Options
+  {
+
+    [Option("tts-rule-fsts", Required = false, Default = "", HelpText = "path to rule.fst")]
+    public string RuleFsts { get; set; }
+
+    [Option("vits-data-dir", Required = false, Default = "", HelpText = "Path to the directory containing dict for espeak-ng.")]
+    public string DataDir { get; set; }
+
+    [Option("vits-length-scale", Required = false, Default = 1, HelpText = "speech speed. Larger->Slower; Smaller->faster")]
+    public float LengthScale { get; set; }
+
+    [Option("vits-noise-scale", Required = false, Default = 0.667f, HelpText = "noise_scale for VITS models")]
+    public float NoiseScale { get; set; }
+
+    [Option("vits-noise-scale-w", Required = false, Default = 0.8f, HelpText = "noise_scale_w for VITS models")]
+    public float NoiseScaleW { get; set; }
+
+    [Option("vits-lexicon", Required = false, Default = "", HelpText = "Path to lexicon.txt")]
+    public string Lexicon { get; set; }
+
+    [Option("vits-tokens", Required = false, Default = "", HelpText = "Path to tokens.txt")]
+    public string Tokens { get; set; }
+
+    [Option("tts-max-num-sentences", Required = false, Default = 1, HelpText = "Maximum number of sentences that we process at a time.")]
+    public int MaxNumSentences { get; set; }
+
+    [Option(Required = false, Default = 0, HelpText = "1 to show debug messages.")]
+    public int Debug { get; set; }
+
+    [Option("vits-model", Required = true, HelpText = "Path to VITS model")]
+    public string Model { get; set; }
+
+    [Option("sid", Required = false, Default = 0, HelpText = "Speaker ID")]
+    public int SpeakerId { get; set; }
+
+    [Option("text", Required = true, HelpText = "Text to synthesize")]
+    public string Text { get; set; }
+
+    [Option("output-filename", Required = true, Default = "./generated.wav", HelpText = "Path to save the generated audio")]
+    public string OutputFilename { get; set; }
+  }
+
+  static void Main(string[] args)
+  {
+    var parser = new CommandLine.Parser(with => with.HelpWriter = null);
+    var parserResult = parser.ParseArguments<Options>(args);
+
+    parserResult
+      .WithParsed<Options>(options => Run(options))
+      .WithNotParsed(errs => DisplayHelp(parserResult, errs));
+  }
+
+  private static void DisplayHelp<T>(ParserResult<T> result, IEnumerable<Error> errs)
+  {
+    string usage = @"
+# vits-aishell3
+
+wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2
+tar xf vits-zh-aishell3.tar.bz2
+
+dotnet run \
+  --vits-model=./vits-zh-aishell3/vits-aishell3.onnx \
+  --vits-tokens=./vits-zh-aishell3/tokens.txt \
+  --vits-lexicon=./vits-zh-aishell3/lexicon.txt \
+  --tts-rule-fsts=./vits-zh-aishell3/rule.fst \
+  --sid=66 \
+  --debug=1 \
+  --output-filename=./aishell3-66.wav \
+  --text=这是一个语音合成测试
+
+# Piper models
+
+wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2
+tar xf vits-piper-en_US-amy-low.tar.bz2
+
+dotnet run \
+  --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \
+  --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \
+  --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \
+  --debug=1 \
+  --output-filename=./amy.wav \
+  --text='This is a text to speech application in dotnet with Next Generation Kaldi'
+
+Please refer to
+https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/index.html
+to download more models.
+";
+
+    var helpText = HelpText.AutoBuild(result, h =>
+    {
+      h.AdditionalNewLineAfterOption = false;
+      h.Heading = usage;
+      h.Copyright = "Copyright (c) 2024 Xiaomi Corporation";
+      return HelpText.DefaultParsingErrorsHandler(result, h);
+    }, e => e);
+    Console.WriteLine(helpText);
+  }
+
+  private static void Run(Options options)
+  {
+    OfflineTtsConfig config = new OfflineTtsConfig();
+    config.Model.Vits.Model = options.Model;
+    config.Model.Vits.Lexicon = options.Lexicon;
+    config.Model.Vits.Tokens = options.Tokens;
+    config.Model.Vits.DataDir = options.DataDir;
+    config.Model.Vits.NoiseScale = options.NoiseScale;
+    config.Model.Vits.NoiseScaleW = options.NoiseScaleW;
+    config.Model.Vits.LengthScale = options.LengthScale;
+    config.Model.NumThreads = 1;
+    config.Model.Debug = options.Debug;
+    config.Model.Provider = "cpu";
+    config.RuleFsts = options.RuleFsts;
+    config.MaxNumSentences = options.MaxNumSentences;
+
+    OfflineTts tts = new OfflineTts(config);
+    float speed = 1.0f / options.LengthScale;
+    int sid = options.SpeakerId;
+    OfflineTtsGeneratedAudio audio = tts.Generate(options.Text, speed, sid);
+    bool ok = audio.SaveToWaveFile(options.OutputFilename);
+
+    if (ok)
+    {
+      Console.WriteLine($"Wrote to {options.OutputFilename} succeeded!");
+    }
+    else
+    {
+      Console.WriteLine($"Failed to write {options.OutputFilename}");
+    }
+  }
+}

dotnet-examples/offline-tts/offline-tts.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <RootNamespace>offline_tts</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="org.k2fsa.sherpa.onnx" Version="*" />
+  </ItemGroup>
+
+</Project>

dotnet-examples/offline-tts/run-aishell3.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ ! -f ./vits-zh-aishell3/vits-aishell3.onnx ]; then
+  wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-zh-aishell3.tar.bz2
+  tar xf vits-zh-aishell3.tar.bz2
+  rm vits-zh-aishell3.tar.bz2
+fi
+
+dotnet run \
+  --vits-model=./vits-zh-aishell3/vits-aishell3.onnx \
+  --vits-tokens=./vits-zh-aishell3/tokens.txt \
+  --vits-lexicon=./vits-zh-aishell3/lexicon.txt \
+  --tts-rule-fsts=./vits-zh-aishell3/rule.fst \
+  --sid=66 \
+  --debug=1 \
+  --output-filename=./aishell3-66.wav \
+  --text="这是一个语音合成测试, 写于公元 2024 年 1 月 28 号, 23点27分，星期天。"

dotnet-examples/offline-tts/run-piper.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+if [ ! -f ./vits-piper-en_US-amy-low/en_US-amy-low.onnx ]; then
+  wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2
+  tar xf vits-piper-en_US-amy-low.tar.bz2
+  rm vits-piper-en_US-amy-low.tar.bz2
+fi
+
+dotnet run \
+  --vits-model=./vits-piper-en_US-amy-low/en_US-amy-low.onnx \
+  --vits-tokens=./vits-piper-en_US-amy-low/tokens.txt \
+  --vits-data-dir=./vits-piper-en_US-amy-low/espeak-ng-data \
+  --debug=1 \
+  --output-filename=./amy.wav \
+  --text="This is a text to speech application in dotnet with Next Generation Kaldi"
+

dotnet-examples/sherpa-onnx.sln

@@ -9,6 +9,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-decode-files", "off
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "speech-recognition-from-microphone", "speech-recognition-from-microphone\speech-recognition-from-microphone.csproj", "{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "offline-tts", "offline-tts\offline-tts.csproj", "{72196886-7143-4043-96E2-BCACEC6C79EB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -30,5 +32,9 @@ Global
 		{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{FE4EA1FF-062A-46B3-B78D-C828FED7B82E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{72196886-7143-4043-96E2-BCACEC6C79EB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{72196886-7143-4043-96E2-BCACEC6C79EB}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

scripts/dotnet/examples/offline-tts.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <RootNamespace>offline_tts</RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <RestoreSources>/tmp/packages;$(RestoreSources);https://api.nuget.org/v3/index.json</RestoreSources>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CommandLineParser" Version="2.9.1" />
+    <PackageReference Include="org.k2fsa.sherpa.onnx" Version="*" />
+  </ItemGroup>
+
+</Project>

scripts/dotnet/offline.cs

@@ -10,6 +10,214 @@ using System;
 namespace SherpaOnnx
 {
 
+  [StructLayout(LayoutKind.Sequential)]
+  public struct OfflineTtsVitsModelConfig
+  {
+    public OfflineTtsVitsModelConfig()
+    {
+      Model = "";
+      Lexicon = "";
+      Tokens = "";
+      DataDir = "";
+
+      NoiseScale = 0.667F;
+      NoiseScaleW = 0.8F;
+      LengthScale = 1.0F;
+    }
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string Model;
+
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string Lexicon;
+
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string Tokens;
+
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string DataDir;
+
+    public float NoiseScale;
+    public float NoiseScaleW;
+    public float LengthScale;
+  }
+
+  [StructLayout(LayoutKind.Sequential)]
+  public struct OfflineTtsModelConfig
+  {
+    public OfflineTtsModelConfig()
+    {
+      Vits = new OfflineTtsVitsModelConfig();
+      NumThreads = 1;
+      Debug = 0;
+      Provider = "cpu";
+    }
+
+    public OfflineTtsVitsModelConfig Vits;
+    public int NumThreads;
+    public int Debug;
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string Provider;
+  }
+
+  public struct OfflineTtsConfig
+  {
+    public OfflineTtsConfig()
+    {
+      Model = new OfflineTtsModelConfig();
+      RuleFsts = "";
+      MaxNumSentences = 1;
+    }
+    public OfflineTtsModelConfig Model;
+
+    [MarshalAs(UnmanagedType.LPStr)]
+    public string RuleFsts;
+
+    public int MaxNumSentences;
+  }
+
+  public class OfflineTtsGeneratedAudio
+  {
+    public OfflineTtsGeneratedAudio(IntPtr p)
+    {
+      _handle = new HandleRef(this, p);
+    }
+
+    public bool SaveToWaveFile(String filename)
+    {
+      Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl));
+      int status = SherpaOnnxWriteWave(impl.Samples, impl.NumSamples, impl.SampleRate, filename);
+      return status == 1;
+    }
+
+    ~OfflineTtsGeneratedAudio()
+    {
+      Cleanup();
+    }
+
+    public void Dispose()
+    {
+      Cleanup();
+      // Prevent the object from being placed on the
+      // finalization queue
+      System.GC.SuppressFinalize(this);
+    }
+
+    private void Cleanup()
+    {
+      SherpaOnnxDestroyOfflineTtsGeneratedAudio(Handle);
+
+      // Don't permit the handle to be used again.
+      _handle = new HandleRef(this, IntPtr.Zero);
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    struct Impl
+    {
+      public IntPtr Samples;
+      public int NumSamples;
+      public int SampleRate;
+    }
+
+    private HandleRef _handle;
+    public IntPtr Handle => _handle.Handle;
+
+    public int NumSamples
+    {
+      get
+      {
+        Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl));
+        return impl.NumSamples;
+      }
+    }
+
+    public int SampleRate
+    {
+      get
+      {
+        Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl));
+        return impl.SampleRate;
+      }
+    }
+
+    public float[] Samples
+    {
+      get
+      {
+        Impl impl = (Impl)Marshal.PtrToStructure(Handle, typeof(Impl));
+
+        float[] samples = new float[impl.NumSamples];
+        Marshal.Copy(impl.Samples, samples, 0, impl.NumSamples);
+        return samples;
+      }
+    }
+
+    [DllImport(Dll.Filename)]
+    private static extern void SherpaOnnxDestroyOfflineTtsGeneratedAudio(IntPtr handle);
+
+    [DllImport(Dll.Filename)]
+    private static extern int SherpaOnnxWriteWave(IntPtr samples, int n, int sample_rate, [MarshalAs(UnmanagedType.LPStr)] string filename);
+  }
+
+  public class OfflineTts : IDisposable
+  {
+    public OfflineTts(OfflineTtsConfig config)
+    {
+      IntPtr h = SherpaOnnxCreateOfflineTts(ref config);
+      _handle = new HandleRef(this, h);
+    }
+
+    public OfflineTtsGeneratedAudio Generate(String text, float speed, int speakerId)
+    {
+      IntPtr p = SherpaOnnxOfflineTtsGenerate(_handle.Handle, text, speakerId, speed);
+      return new OfflineTtsGeneratedAudio(p);
+    }
+
+    public void Dispose()
+    {
+      Cleanup();
+      // Prevent the object from being placed on the
+      // finalization queue
+      System.GC.SuppressFinalize(this);
+    }
+
+    ~OfflineTts()
+    {
+      Cleanup();
+    }
+
+    private void Cleanup()
+    {
+      SherpaOnnxDestroyOfflineTts(_handle.Handle);
+
+      // Don't permit the handle to be used again.
+      _handle = new HandleRef(this, IntPtr.Zero);
+    }
+
+    private HandleRef _handle;
+
+    public int SampleRate
+    {
+      get
+      {
+        return SherpaOnnxOfflineTtsSampleRate(_handle.Handle);
+      }
+    }
+
+    [DllImport(Dll.Filename)]
+    private static extern IntPtr SherpaOnnxCreateOfflineTts(ref OfflineTtsConfig config);
+
+    [DllImport(Dll.Filename)]
+    private static extern void SherpaOnnxDestroyOfflineTts(IntPtr handle);
+
+    [DllImport(Dll.Filename)]
+    private static extern int SherpaOnnxOfflineTtsSampleRate(IntPtr handle);
+
+    [DllImport(Dll.Filename)]
+    private static extern IntPtr SherpaOnnxOfflineTtsGenerate(IntPtr handle, [MarshalAs(UnmanagedType.LPStr)] string text, int sid, float speed);
+  }
+
+
+
   [StructLayout(LayoutKind.Sequential)]
   public struct OfflineTransducerModelConfig
   {

scripts/dotnet/run.sh

@@ -15,12 +15,23 @@ pushd /tmp
 
 mkdir -p linux macos windows
 
+# You can pre-download the required wheels to /tmp
+src_dir=/tmp
+
+linux_wheel=$src_dir/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+macos_wheel=$src_dir/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-macosx_10_14_x86_64.whl
+windows_wheel=$src_dir/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl
+
 if [ ! -f /tmp/linux/libsherpa-onnx-core.so ]; then
   echo "---linux x86_64---"
   cd linux
-  mkdir wheel
+  mkdir -p wheel
   cd wheel
+  if [ -f $linux_wheel ]; then
+    cp -v $linux_wheel .
+  else
     curl -OL https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+  fi
   unzip sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
   cp -v sherpa_onnx/lib/*.so* ../
   cd ..
@@ -36,9 +47,13 @@ fi
 if [ ! -f /tmp/macos/libsherpa-onnx-core.dylib ]; then
   echo "---macOS x86_64---"
   cd macos
-  mkdir wheel
+  mkdir -p wheel
   cd wheel
+  if [ -f $macos_wheel  ]; then
+    cp -v $macos_wheel .
+  else
     curl -OL https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-macosx_10_14_x86_64.whl
+  fi
   unzip sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-macosx_10_14_x86_64.whl
   cp -v sherpa_onnx/lib/*.dylib ../
 
@@ -57,9 +72,13 @@ fi
 if [ ! -f /tmp/windows/libsherpa-onnx-core.dll ]; then
   echo "---windows x64---"
   cd windows
-  mkdir wheel
+  mkdir -p wheel
   cd wheel
+  if [ -f $windows_wheel ]; then
+    cp -v $windows_wheel .
+  else
     curl -OL https://huggingface.co/csukuangfj/sherpa-onnx-wheels/resolve/main/sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl
+  fi
   unzip sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win_amd64.whl
   cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.dll ../
   cp -v sherpa_onnx-${SHERPA_ONNX_VERSION}.data/data/bin/*.lib ../
@@ -101,5 +120,5 @@ popd
 
 ls -lh packages
 
-mkdir /tmp/packages
+mkdir -p /tmp/packages
 cp -v packages/*.nupkg /tmp/packages

sherpa-onnx/csrc/online-stream.h

@@ -18,7 +18,7 @@
 
 namespace sherpa_onnx {
 
-class TransducerKeywordResult;
+struct TransducerKeywordResult;
 class OnlineStream {
  public:
   explicit OnlineStream(const FeatureExtractorConfig &config = {},