Add WebAssembly (WASM) for speech enhancement GTCRN models (#2002)

2025-03-13 18:35:03 +08:00
parent 6a97f8adcf
commit d320fdf65e
29 changed files with 1175 additions and 161 deletions
--- a/wasm/tts/app-tts.js
+++ b/wasm/tts/app-tts.js
@@ -1,5 +1,4 @@
 const generateBtn = document.getElementById('generateBtn');
-const hint = document.getElementById('hint');
 const speakerIdLabel = document.getElementById('speakerIdLabel');
 const speakerIdInput = document.getElementById('speakerId');
 const speedInput = document.getElementById('speed');
@@ -11,13 +10,41 @@ speedValue.innerHTML = speedInput.value;

 let index = 0;

-
 let tts = null;

 let audioCtx = null;

-
 Module = {};
+
+// https://emscripten.org/docs/api_reference/module.html#Module.locateFile
+Module.locateFile = function(path, scriptDirectory = '') {
+  console.log(`path: ${path}, scriptDirectory: ${scriptDirectory}`);
+  return scriptDirectory + path;
+};
+
+// https://emscripten.org/docs/api_reference/module.html#Module.locateFile
+Module.setStatus = function(status) {
+  console.log(`status ${status}`);
+  const statusElement = document.getElementById('status');
+  if (status == "Running...") {
+    status = 'Model downloaded. Initializing text to speech model...'
+  }
+  statusElement.textContent = status;
+  if (status === '') {
+    statusElement.style.display = 'none';
+    // statusElement.parentNode.removeChild(statusElement);
+
+    document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
+      tabContentElement.classList.remove('loading');
+    });
+  } else {
+    statusElement.style.display = 'block';
+    document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
+      tabContentElement.classList.add('loading');
+    });
+  }
+};
+
 Module.onRuntimeInitialized = function() {
  console.log('Model files downloaded!');

@@ -27,17 +54,10 @@ Module.onRuntimeInitialized = function() {
    speakerIdLabel.innerHTML = `Speaker ID (0 - ${tts.numSpeakers - 1}):`;
  }

-  hint.innerText =
-      'Initialized! Please enter text and click the Generate button.';
-
-
-
  generateBtn.disabled = false;
 };

-speedInput.oninput = function() {
-  speedValue.innerHTML = this.value;
-};
+speedInput.oninput = function() { speedValue.innerHTML = this.value; };

 generateBtn.onclick = function() {
  let speakerId = speakerIdInput.value;
@@ -69,12 +89,12 @@ generateBtn.onclick = function() {
  console.log('text', text);

  let audio =
-      tts.generate({text: text, sid: speakerId, speed: speedInput.value});
+      tts.generate({text : text, sid : speakerId, speed : speedInput.value});

  console.log(audio.samples.length, audio.sampleRate);

  if (!audioCtx) {
-    audioCtx = new AudioContext({sampleRate: tts.sampleRate});
+    audioCtx = new AudioContext({sampleRate : tts.sampleRate});
  }

  const buffer = audioCtx.createBuffer(1, audio.samples.length, tts.sampleRate);
@@ -155,22 +175,22 @@ function toWav(floatSamples, sampleRate) {

  // http://soundfile.sapp.org/doc/WaveFormat/
  //                   F F I R
-  view.setUint32(0, 0x46464952, true);               // chunkID
-  view.setUint32(4, 36 + samples.length * 2, true);  // chunkSize
+  view.setUint32(0, 0x46464952, true);              // chunkID
+  view.setUint32(4, 36 + samples.length * 2, true); // chunkSize
  //                   E V A W
-  view.setUint32(8, 0x45564157, true);  // format
-                                        //
+  view.setUint32(8, 0x45564157, true); // format
+                                       //
  //                      t m f
-  view.setUint32(12, 0x20746d66, true);          // subchunk1ID
-  view.setUint32(16, 16, true);                  // subchunk1Size, 16 for PCM
-  view.setUint32(20, 1, true);                   // audioFormat, 1 for PCM
-  view.setUint16(22, 1, true);                   // numChannels: 1 channel
-  view.setUint32(24, sampleRate, true);          // sampleRate
-  view.setUint32(28, sampleRate * 2, true);      // byteRate
-  view.setUint16(32, 2, true);                   // blockAlign
-  view.setUint16(34, 16, true);                  // bitsPerSample
-  view.setUint32(36, 0x61746164, true);          // Subchunk2ID
-  view.setUint32(40, samples.length * 2, true);  // subchunk2Size
+  view.setUint32(12, 0x20746d66, true);         // subchunk1ID
+  view.setUint32(16, 16, true);                 // subchunk1Size, 16 for PCM
+  view.setUint32(20, 1, true);                  // audioFormat, 1 for PCM
+  view.setUint16(22, 1, true);                  // numChannels: 1 channel
+  view.setUint32(24, sampleRate, true);         // sampleRate
+  view.setUint32(28, sampleRate * 2, true);     // byteRate
+  view.setUint16(32, 2, true);                  // blockAlign
+  view.setUint16(34, 16, true);                 // bitsPerSample
+  view.setUint32(36, 0x61746164, true);         // Subchunk2ID
+  view.setUint32(40, samples.length * 2, true); // subchunk2Size

  let offset = 44;
  for (let i = 0; i < samples.length; ++i) {
@@ -178,5 +198,5 @@ function toWav(floatSamples, sampleRate) {
    offset += 2;
  }

-  return new Blob([view], {type: 'audio/wav'});
+  return new Blob([ view ], {type : 'audio/wav'});
 }
--- a/wasm/tts/assets/README.md
+++ b/wasm/tts/assets/README.md
@@ -5,7 +5,7 @@ https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
 to download a model.

 The following is an example:
-```
+```bash
 cd sherpa-onnx/wasm/tts/assets

 wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2
--- a/wasm/tts/index.html
+++ b/wasm/tts/index.html
@@ -11,34 +11,70 @@
    textarea {
      width:100%;
    }
+    .loading {
+      display: none !important;
+    }
  </style>
 </head>

-<body>
+<body style="font-family: 'Source Sans Pro', sans-serif; background-color: #f9fafb; color: #333; display: flex; flex-direction: column; align-items: center; height: 100vh; margin: 0;">
  <h1>
    Next-gen Kaldi + WebAssembly<br/>
    Text-to-speech Demo with <a href="https://github.com/k2-fsa/sherpa-onnx">sherpa-onnx</a>
  </h1>
-  <div>
-    <span id="hint">Loading model ... ...</span>
-    <br/>
-    <br/>
-    <label for="speakerId" id="speakerIdLabel">Speaker ID: </label>
-    <input type="text" id="speakerId" name="speakerId" value="0" />
-    <br/>
-    <br/>
-    <label for="speed" id="speedLabel">Speed: </label>
-    <input type="range" id="speed" name="speed" min="0.4" max="3.5" step="0.1" value="1.0" />
-    <span id="speedValue"></span>
-    <br/>
-    <br/>
-    <textarea id="text" rows="10" placeholder="Please enter your text here and click the Generate button"></textarea>
-    <br/>
-    <br/>
-    <button id="generateBtn" disabled>Generate</button>
+
+  <div style="width: 100%; max-width: 900px; background: #fff; padding: 1.5rem; border-radius: 8px; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); flex: 1;">
+    <div id="status">Loading...</div>
+
+    <div id="singleAudioContent" class="tab-content loading">
+      <label for="speakerId" id="speakerIdLabel">Speaker ID: </label>
+      <input type="text" id="speakerId" name="speakerId" value="0" />
+      <br/>
+      <br/>
+      <label for="speed" id="speedLabel">Speed: </label>
+      <input type="range" id="speed" name="speed" min="0.4" max="3.5" step="0.1" value="1.0" />
+      <span id="speedValue"></span>
+      <br/>
+      <br/>
+      <textarea id="text" rows="10" placeholder="Please enter your text here and click the Generate button"></textarea>
+      <br/>
+      <br/>
+      <button id="generateBtn" disabled>Generate</button>
+    </div>
+
+    <section flex="1" overflow="auto" id="sound-clips">
+    </section>
  </div>
-  <section flex="1" overflow="auto" id="sound-clips">
-  </section>
+
+  <!-- Footer Section -->
+  <div style="width: 100%; max-width: 900px; margin-top: 1.5rem; background: #fff; padding: 1.5rem; border-radius: 8px; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); text-align: left; font-size: 0.9rem; color: #6c757d;">
+    <h3>Description</h3>
+    <ul>
+      <li>Everything is <strong>open-sourced.</strong> <a href="https://github.com/k2-fsa/sherpa-onnx">code</a></li>
+      <li>If you have any issues, please either <a href="https://github.com/k2-fsa/sherpa-onnx/issues">file a ticket</a> or contact us via</li>
+        <ul>
+          <li><a href="https://k2-fsa.github.io/sherpa/social-groups.html#wechat">WeChat group</a></li>
+          <li><a href="https://k2-fsa.github.io/sherpa/social-groups.html#qq">QQ group</a></li>
+          <li><a href="https://k2-fsa.github.io/sherpa/social-groups.html#bilibili-b">Bilibili</a></li>
+        </ul>
+    </ul>
+    <h3>About This Demo</h3>
+    <ul>
+      <li><strong>Private and Secure:</strong> All processing is done locally on your device (CPU) within your browser with a single thread. No server is involved, ensuring privacy and security. You can disconnect from the Internet once this page is loaded.</li>
+      <li><strong>Efficient Resource Usage:</strong> No GPU is required, leaving system resources available for webLLM analysis.</li>
+    </ul>
+    <h3>Latest Update</h3>
+    <ul>
+      <li>Update UI.</li>
+      <li>First working version.</li>
+    </ul>
+
+    <h3>Acknowledgement</h3>
+    <ul>
+      <li>We refer to <a href="https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Wasm">https://huggingface.co/spaces/Banafo/Kroko-Streaming-ASR-Wasm</a> for the UI part.</li>
+    </ul>
+  </div>
+

  <script src="app-tts.js"></script>
  <script src="sherpa-onnx-tts.js"></script>
--- a/wasm/tts/sherpa-onnx-tts.js
+++ b/wasm/tts/sherpa-onnx-tts.js
@@ -263,7 +263,7 @@ function initSherpaOnnxOfflineTtsModelConfig(config, Module) {

  const providerLen = Module.lengthBytesUTF8(config.provider || 'cpu') + 1;
  const buffer = Module._malloc(providerLen);
-  Module.stringToUTF8(config.provider, buffer, providerLen);
+  Module.stringToUTF8(config.provider || 'cpu', buffer, providerLen);
  Module.setValue(ptr + offset, buffer, 'i8*');
  offset += 4;