Support WebAssembly for text-to-speech (#577)

2024-02-08 23:39:12 +08:00
parent 324a265523
commit d771762868
20 changed files with 887 additions and 68 deletions
--- a/wasm/CMakeLists.txt
+++ b/wasm/CMakeLists.txt
@@ -0,0 +1,59 @@
+if(NOT $ENV{SHERPA_ONNX_IS_USING_BUILD_WASM_SH})
+  message(FATAL_ERROR "Please use ./build-wasm.sh to build for wasm")
+endif()
+
+if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/decoder_jit_trace-pnnx.ncnn.bin")
+  message(WARNING "${CMAKE_CURRENT_SOURCE_DIR}/assets/decoder_jit_trace-pnnx.ncnn.bin does not exist")
+  # message(FATAL_ERROR "Please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md before you continue")
+endif()
+
+set(exported_functions
+  MyPrint
+  SherpaOnnxCreateOfflineTts
+  SherpaOnnxDestroyOfflineTts
+  SherpaOnnxDestroyOfflineTtsGeneratedAudio
+  SherpaOnnxOfflineTtsGenerate
+  SherpaOnnxOfflineTtsGenerateWithCallback
+  SherpaOnnxOfflineTtsNumSpeakers
+  SherpaOnnxOfflineTtsSampleRate
+  SherpaOnnxWriteWave
+)
+set(mangled_exported_functions)
+foreach(x IN LISTS exported_functions)
+  list(APPEND mangled_exported_functions "_${x}")
+endforeach()
+list(JOIN mangled_exported_functions "," all_exported_functions)
+
+
+include_directories(${CMAKE_SOURCE_DIR})
+set(MY_FLAGS " -s FORCE_FILESYSTEM=1 -s INITIAL_MEMORY=512MB -s ALLOW_MEMORY_GROWTH=1")
+string(APPEND MY_FLAGS " -sSTACK_SIZE=10485760 ") # 10MB
+string(APPEND MY_FLAGS " -sEXPORTED_FUNCTIONS=[_CopyHeap,_malloc,_free,${all_exported_functions}] ")
+string(APPEND MY_FLAGS "--preload-file ${CMAKE_CURRENT_SOURCE_DIR}/assets@. ")
+string(APPEND MY_FLAGS " -sEXPORTED_RUNTIME_METHODS=['ccall','stringToUTF8','setValue','getValue'] ")
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${MY_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${MY_FLAGS}")
+set(CMAKE_EXECUTBLE_LINKER_FLAGS "${CMAKE_EXECUTBLE_LINKER_FLAGS} ${MY_FLAGS}")
+
+if (NOT CMAKE_EXECUTABLE_SUFFIX STREQUAL ".js")
+  message(FATAL_ERROR "The default suffix for building executables should be .js!")
+endif()
+# set(CMAKE_EXECUTABLE_SUFFIX ".html")
+
+add_executable(sherpa-onnx-wasm-main sherpa-onnx-wasm-main.cc)
+target_link_libraries(sherpa-onnx-wasm-main sherpa-onnx-c-api)
+install(TARGETS sherpa-onnx-wasm-main DESTINATION bin/wasm)
+
+install(
+  FILES
+    "$<TARGET_FILE_DIR:sherpa-onnx-wasm-main>/sherpa-onnx-wasm-main.js"
+    "index.html"
+    "sherpa-onnx.js"
+    "app.js"
+    "$<TARGET_FILE_DIR:sherpa-onnx-wasm-main>/sherpa-onnx-wasm-main.wasm"
+    "$<TARGET_FILE_DIR:sherpa-onnx-wasm-main>/sherpa-onnx-wasm-main.data"
+    # "$<TARGET_FILE_DIR:sherpa-onnx-wasm-main>/sherpa-onnx-wasm-main.html"
+  DESTINATION
+    bin/wasm
+)
--- a/wasm/app.js
+++ b/wasm/app.js
@@ -0,0 +1,182 @@
+const generateBtn = document.getElementById('generateBtn');
+const hint = document.getElementById('hint');
+const speakerIdLabel = document.getElementById('speakerIdLabel');
+const speakerIdInput = document.getElementById('speakerId');
+const speedInput = document.getElementById('speed');
+const speedValue = document.getElementById('speedValue');
+const textArea = document.getElementById('text');
+const soundClips = document.getElementById('sound-clips');
+
+speedValue.innerHTML = speedInput.value;
+
+let index = 0;
+
+
+let tts = null;
+
+let audioCtx = null;
+
+
+Module = {};
+Module.onRuntimeInitialized = function() {
+  console.log('Model files downloaded!');
+
+  console.log('Initializing tts ......');
+  tts = initSherpaOnnxOfflineTts()
+  if (tts.numSpeakers > 1) {
+    speakerIdLabel.innerHTML = `Speaker ID (0 - ${tts.numSpeakers - 1}):`;
+  }
+
+  hint.innerText =
+      'Initialized! Please enter text and click the Generate button.';
+
+
+
+  generateBtn.disabled = false;
+};
+
+speedInput.oninput = function() {
+  speedValue.innerHTML = this.value;
+};
+
+generateBtn.onclick = function() {
+  let speakerId = speakerIdInput.value;
+  if (speakerId.trim().length == 0) {
+    alert('Please input a speakerId');
+    return;
+  }
+
+  if (!speakerId.match(/^\d+$/)) {
+    alert(`Input speakerID ${
+        speakerId} is not a number.\nPlease enter a number between 0 and ${
+        tts.numSpeakers - 1}`);
+    return;
+  }
+  speakerId = parseInt(speakerId, 10);
+  if (speakerId > tts.numSpeakers - 1) {
+    alert(`Pleaser enter a number between 0 and ${tts.numSpeakers - 1}`);
+    return;
+  }
+
+  let text = textArea.value.trim();
+  if (text.length == 0) {
+    alert('Please input a non-blank text');
+    return;
+  }
+
+  console.log('speakerId', speakerId);
+  console.log('speed', speedInput.value);
+  console.log('text', text);
+
+  let audio =
+      tts.generate({text: text, sid: speakerId, speed: speedInput.value});
+
+  console.log(audio.samples.length, audio.sampleRate);
+
+  if (!audioCtx) {
+    audioCtx = new AudioContext({sampleRate: tts.sampleRate});
+  }
+
+  const buffer = audioCtx.createBuffer(1, audio.samples.length, tts.sampleRate);
+
+  const ptr = buffer.getChannelData(0);
+  for (let i = 0; i < audio.samples.length; i++) {
+    ptr[i] = audio.samples[i];
+  }
+  const source = audioCtx.createBufferSource();
+  source.buffer = buffer;
+  source.connect(audioCtx.destination);
+  source.start();
+
+  createAudioTag(audio);
+};
+
+function createAudioTag(generateAudio) {
+  const blob = toWav(generateAudio.samples, generateAudio.sampleRate);
+
+  const text = textArea.value.trim().substring(0, 100);
+  const clipName = `${index} ${text} ...`;
+  index += 1;
+
+  const clipContainer = document.createElement('article');
+  const clipLabel = document.createElement('p');
+  const audio = document.createElement('audio');
+  const deleteButton = document.createElement('button');
+  clipContainer.classList.add('clip');
+  audio.setAttribute('controls', '');
+  deleteButton.textContent = 'Delete';
+  deleteButton.className = 'delete';
+
+  clipLabel.textContent = clipName;
+
+  clipContainer.appendChild(audio);
+
+  clipContainer.appendChild(clipLabel);
+  clipContainer.appendChild(deleteButton);
+  soundClips.appendChild(clipContainer);
+
+  audio.controls = true;
+
+  const audioURL = window.URL.createObjectURL(blob);
+  audio.src = audioURL;
+
+  deleteButton.onclick = function(e) {
+    let evtTgt = e.target;
+    evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode);
+  };
+
+  clipLabel.onclick = function() {
+    const existingName = clipLabel.textContent;
+    const newClipName = prompt('Enter a new name for your sound clip?');
+    if (newClipName === null) {
+      clipLabel.textContent = existingName;
+    } else {
+      clipLabel.textContent = newClipName;
+    }
+  };
+}
+
+// this function is copied/modified from
+// https://gist.github.com/meziantou/edb7217fddfbb70e899e
+function toWav(floatSamples, sampleRate) {
+  let samples = new Int16Array(floatSamples.length);
+  for (let i = 0; i < samples.length; ++i) {
+    let s = floatSamples[i];
+    if (s >= 1)
+      s = 1;
+    else if (s <= -1)
+      s = -1;
+
+    samples[i] = s * 32767;
+  }
+
+  let buf = new ArrayBuffer(44 + samples.length * 2);
+  var view = new DataView(buf);
+
+  // http://soundfile.sapp.org/doc/WaveFormat/
+  //                   F F I R
+  view.setUint32(0, 0x46464952, true);               // chunkID
+  view.setUint32(4, 36 + samples.length * 2, true);  // chunkSize
+  //                   E V A W
+  view.setUint32(8, 0x45564157, true);  // format
+                                        //
+  //                      t m f
+  view.setUint32(12, 0x20746d66, true);          // subchunk1ID
+  view.setUint32(16, 16, true);                  // subchunk1Size, 16 for PCM
+  view.setUint32(20, 1, true);                   // audioFormat, 1 for PCM
+  view.setUint16(22, 1, true);                   // numChannels: 1 channel
+  view.setUint32(24, sampleRate, true);          // sampleRate
+  view.setUint32(28, sampleRate * 2, true);      // byteRate
+  view.setUint16(32, 2, true);                   // blockAlign
+  view.setUint16(34, 16, true);                  // bitsPerSample
+  view.setUint32(36, 0x61746164, true);          // Subchunk2ID
+  view.setUint32(40, samples.length * 2, true);  // subchunk2Size
+
+  let offset = 44;
+  for (let i = 0; i < samples.length; ++i) {
+    view.setInt16(offset, samples[i], true);
+    offset += 2;
+  }
+
+  return new Blob([view], {type: 'audio/wav'});
+}
--- a/wasm/assets/.gitignore
+++ b/wasm/assets/.gitignore
@@ -0,0 +1,4 @@
+*.onnx
+*.txt
+espeak-ng-data
+
--- a/wasm/assets/README.md
+++ b/wasm/assets/README.md
@@ -0,0 +1,32 @@
+# Introduction
+
+Please refer to
+https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+to download a model.
+
+The following is an example:
+```
+cd sherpa-onnx/wasm/tts/assets
+
+wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2
+tar xf vits-piper-en_US-libritts_r-medium.tar.bz2
+rm vits-piper-en_US-libritts_r-medium.tar.bz2
+mv vits-piper-en_US-libritts_r-medium/en_US-libritts_r-medium.onnx ./model.onnx
+mv vits-piper-en_US-libritts_r-medium/tokens.txt ./
+mv vits-piper-en_US-libritts_r-medium/espeak-ng-data ./
+rm -rf vits-piper-en_US-libritts_r-medium
+```
+
+You should have the following files in `assets` before you can run
+`build-wasm-simd.sh`
+
+```
+assets fangjun$ tree -L 1
+.
+├── README.md
+├── espeak-ng-data
+├── mode.onnx
+└── tokens.txt
+
+1 directory, 3 files
+```
--- a/wasm/index.html
+++ b/wasm/index.html
@@ -0,0 +1,46 @@
+<html lang="en">
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width" />
+  <title>Next-gen Kaldi WebAssembly with sherpa-onnx for Text-to-speech</title>
+  <style>
+    h1,div {
+      text-align: center;
+    }
+    textarea {
+      width:100%;
+    }
+  </style>
+</head>
+
+<body>
+  <h1>
+    Next-gen Kaldi + WebAssembly<br/>
+    Text-to-speech Demo with <a href="https://github.com/k2-fsa/sherpa-onnx">sherpa-onnx</a>
+  </h1>
+  <div>
+    <span id="hint">Loading model ... ...</span>
+    <br/>
+    <br/>
+    <label for="speakerId" id="speakerIdLabel">Speaker ID: </label>
+    <input type="text" id="speakerId" name="speakerId" value="0" />
+    <br/>
+    <br/>
+    <label for="speed" id="speedLabel">Speed: </label>
+    <input type="range" id="speed" name="speed" min="0.4" max="3.5" step="0.1" value="1.0" />
+    <span id="speedValue"></span>
+    <br/>
+    <br/>
+    <textarea id="text" rows="10" placeholder="Please enter your text here and click the Generate button"></textarea>
+    <br/>
+    <br/>
+    <button id="generateBtn" disabled>Generate</button>
+  </div>
+  <section flex="1" overflow="auto" id="sound-clips">
+  </section>
+
+  <script src="app.js"></script>
+  <script src="sherpa-onnx.js"></script>
+  <script src="sherpa-onnx-wasm-main.js"></script>
+</body>
--- a/wasm/sherpa-onnx-wasm-main.cc
+++ b/wasm/sherpa-onnx-wasm-main.cc
@@ -0,0 +1,49 @@
+// wasm/sherpa-onnx-wasm-main.cc
+//
+// Copyright (c)  2024  Xiaomi Corporation
+#include <stdio.h>
+
+#include <algorithm>
+#include <memory>
+
+#include "sherpa-onnx/c-api/c-api.h"
+
+// see also
+// https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html
+
+extern "C" {
+
+static_assert(sizeof(SherpaOnnxOfflineTtsVitsModelConfig) == 7 * 4, "");
+static_assert(sizeof(SherpaOnnxOfflineTtsModelConfig) ==
+                  sizeof(SherpaOnnxOfflineTtsVitsModelConfig) + 3 * 4,
+              "");
+static_assert(sizeof(SherpaOnnxOfflineTtsConfig) ==
+                  sizeof(SherpaOnnxOfflineTtsModelConfig) + 2 * 4,
+              "");
+
+void MyPrint(SherpaOnnxOfflineTtsConfig *tts_config) {
+  auto tts_model_config = &tts_config->model;
+  auto vits_model_config = &tts_model_config->vits;
+  fprintf(stdout, "----------vits model config----------\n");
+  fprintf(stdout, "model: %s\n", vits_model_config->model);
+  fprintf(stdout, "lexicon: %s\n", vits_model_config->lexicon);
+  fprintf(stdout, "tokens: %s\n", vits_model_config->tokens);
+  fprintf(stdout, "data_dir: %s\n", vits_model_config->data_dir);
+  fprintf(stdout, "noise scale: %.3f\n", vits_model_config->noise_scale);
+  fprintf(stdout, "noise scale w: %.3f\n", vits_model_config->noise_scale_w);
+  fprintf(stdout, "length scale: %.3f\n", vits_model_config->length_scale);
+
+  fprintf(stdout, "----------tts model config----------\n");
+  fprintf(stdout, "num threads: %d\n", tts_model_config->num_threads);
+  fprintf(stdout, "debug: %d\n", tts_model_config->debug);
+  fprintf(stdout, "provider: %s\n", tts_model_config->provider);
+
+  fprintf(stdout, "----------tts config----------\n");
+  fprintf(stdout, "rule_fsts: %s\n", tts_config->rule_fsts);
+  fprintf(stdout, "max num sentences: %d\n", tts_config->max_num_sentences);
+}
+
+void CopyHeap(const char *src, int32_t num_bytes, char *dst) {
+  std::copy(src, src + num_bytes, dst);
+}
+}
--- a/wasm/sherpa-onnx.js
+++ b/wasm/sherpa-onnx.js
@@ -0,0 +1,180 @@
+
+function freeConfig(config) {
+  if ('buffer' in config) {
+    _free(config.buffer);
+  }
+
+  if ('config' in config) {
+    freeConfig(config.config)
+  }
+
+  _free(config.ptr);
+}
+
+// The user should free the returned pointers
+function initSherpaOnnxOfflineTtsVitsModelConfig(config) {
+  let modelLen = lengthBytesUTF8(config.model) + 1;
+  let lexiconLen = lengthBytesUTF8(config.lexicon) + 1;
+  let tokensLen = lengthBytesUTF8(config.tokens) + 1;
+  let dataDirLen = lengthBytesUTF8(config.dataDir) + 1;
+
+  let n = modelLen + lexiconLen + tokensLen + dataDirLen;
+
+  let buffer = _malloc(n);
+
+  let len = 7 * 4;
+  let ptr = _malloc(len);
+
+  let offset = 0;
+  stringToUTF8(config.model, buffer + offset, modelLen);
+  offset += modelLen;
+
+  stringToUTF8(config.lexicon, buffer + offset, lexiconLen);
+  offset += lexiconLen;
+
+  stringToUTF8(config.tokens, buffer + offset, tokensLen);
+  offset += tokensLen;
+
+  stringToUTF8(config.dataDir, buffer + offset, dataDirLen);
+  offset += dataDirLen;
+
+  offset = 0;
+  setValue(ptr, buffer + offset, 'i8*');
+  offset += modelLen;
+
+  setValue(ptr + 4, buffer + offset, 'i8*');
+  offset += lexiconLen;
+
+  setValue(ptr + 8, buffer + offset, 'i8*');
+  offset += tokensLen;
+
+  setValue(ptr + 12, buffer + offset, 'i8*');
+  offset += dataDirLen;
+
+  setValue(ptr + 16, config.noiseScale, 'float');
+  setValue(ptr + 20, config.noiseScaleW, 'float');
+  setValue(ptr + 24, config.lengthScale, 'float');
+
+  return {
+    buffer: buffer, ptr: ptr, len: len,
+  }
+}
+
+function initSherpaOnnxOfflineTtsModelConfig(config) {
+  let vitsModelConfig =
+      initSherpaOnnxOfflineTtsVitsModelConfig(config.offlineTtsVitsModelConfig);
+
+  let len = vitsModelConfig.len + 3 * 4;
+  let ptr = _malloc(len);
+
+  let offset = 0;
+  _CopyHeap(vitsModelConfig.ptr, vitsModelConfig.len, ptr + offset);
+  offset += vitsModelConfig.len;
+
+  setValue(ptr + offset, config.numThreads, 'i32');
+  offset += 4;
+
+  setValue(ptr + offset, config.debug, 'i32');
+  offset += 4;
+
+  let providerLen = lengthBytesUTF8(config.provider) + 1;
+  let buffer = _malloc(providerLen);
+  stringToUTF8(config.provider, buffer, providerLen);
+  setValue(ptr + offset, buffer, 'i8*');
+
+  return {
+    buffer: buffer, ptr: ptr, len: len, config: vitsModelConfig,
+  }
+}
+
+function initSherpaOnnxOfflineTtsConfig(config) {
+  let modelConfig =
+      initSherpaOnnxOfflineTtsModelConfig(config.offlineTtsModelConfig);
+  let len = modelConfig.len + 2 * 4;
+  let ptr = _malloc(len);
+
+  let offset = 0;
+  _CopyHeap(modelConfig.ptr, modelConfig.len, ptr + offset);
+  offset += modelConfig.len;
+
+  let ruleFstsLen = lengthBytesUTF8(config.ruleFsts) + 1;
+  let buffer = _malloc(ruleFstsLen);
+  stringToUTF8(config.ruleFsts, buffer, ruleFstsLen);
+  setValue(ptr + offset, buffer, 'i8*');
+  offset += 4;
+
+  setValue(ptr + offset, config.maxNumSentences, 'i32');
+
+  return {
+    buffer: buffer, ptr: ptr, len: len, config: modelConfig,
+  }
+}
+
+class OfflineTts {
+  constructor(configObj) {
+    let config = initSherpaOnnxOfflineTtsConfig(configObj)
+    let handle = _SherpaOnnxCreateOfflineTts(config.ptr);
+
+    freeConfig(config);
+
+    this.handle = handle;
+    this.sampleRate = _SherpaOnnxOfflineTtsSampleRate(this.handle);
+    this.numSpeakers = _SherpaOnnxOfflineTtsNumSpeakers(this.handle);
+  }
+
+  free() {
+    _SherpaOnnxDestroyOfflineTts(this.handle);
+    this.handle = 0
+  }
+
+  // {
+  //   text: "hello",
+  //   sid: 1,
+  //   speed: 1.0
+  // }
+  generate(config) {
+    let textLen = lengthBytesUTF8(config.text) + 1;
+    let textPtr = _malloc(textLen);
+    stringToUTF8(config.text, textPtr, textLen);
+
+    let h = _SherpaOnnxOfflineTtsGenerate(
+        this.handle, textPtr, config.sid, config.speed);
+
+    let numSamples = HEAP32[h / 4 + 1];
+    let sampleRate = HEAP32[h / 4 + 2];
+
+    let samplesPtr = HEAP32[h / 4] / 4;
+    let samples = new Float32Array(numSamples);
+    for (let i = 0; i < numSamples; i++) {
+      samples[i] = HEAPF32[samplesPtr + i];
+    }
+
+    _SherpaOnnxDestroyOfflineTtsGeneratedAudio(h);
+    return {samples: samples, sampleRate: sampleRate};
+  }
+}
+
+function initSherpaOnnxOfflineTts() {
+  let offlineTtsVitsModelConfig = {
+    model: './model.onnx',
+    lexicon: '',
+    tokens: './tokens.txt',
+    dataDir: './espeak-ng-data',
+    noiseScale: 0.667,
+    noiseScaleW: 0.8,
+    lengthScale: 1.0,
+  };
+  let offlineTtsModelConfig = {
+    offlineTtsVitsModelConfig: offlineTtsVitsModelConfig,
+    numThreads: 1,
+    debug: 1,
+    provider: 'cpu',
+  };
+  let offlineTtsConfig = {
+    offlineTtsModelConfig: offlineTtsModelConfig,
+    ruleFsts: '',
+    maxNumSentences: 1,
+  }
+
+  return new OfflineTts(offlineTtsConfig);
+}