EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add Javascript (WebAssembly) API for Dolphin CTC models (#2093)

Browse Source
This commit is contained in:
Fangjun Kuang
2025-04-03 15:02:06 +08:00
committed by GitHub
parent 74f402e490
commit 639ad1744f
9 changed files with 172 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
wasm/asr/sherpa-onnx-asr.js
View File

@@ -39,6 +39,10 @@ function freeConfig(config, Module) {
freeConfig(config.fireRedAsr, Module)
}
if ('dolphin' in config) {
freeConfig(config.dolphin, Module)
}
if ('moonshine' in config) {
freeConfig(config.moonshine, Module)
}
@@ -562,6 +566,23 @@ function initSherpaOnnxOfflineNemoEncDecCtcModelConfig(config, Module) {
}
}
function initSherpaOnnxOfflineDolphinModelConfig(config, Module) {
const n = Module.lengthBytesUTF8(config.model || '') + 1;
const buffer = Module._malloc(n);
const len = 1 * 4; // 1 pointer
const ptr = Module._malloc(len);
Module.stringToUTF8(config.model || '', buffer, n);
Module.setValue(ptr, buffer, 'i8*');
return {
buffer: buffer, ptr: ptr, len: len,
}
}
function initSherpaOnnxOfflineWhisperModelConfig(config, Module) {
const encoderLen = Module.lengthBytesUTF8(config.encoder || '') + 1;
const decoderLen = Module.lengthBytesUTF8(config.decoder || '') + 1;
@@ -769,6 +790,12 @@ function initSherpaOnnxOfflineModelConfig(config, Module) {
};
}
if (!('dolphin' in config)) {
config.dolphin = {
model: '',
};
}
if (!('whisper' in config)) {
config.whisper = {
encoder: '',
@@ -832,8 +859,12 @@ function initSherpaOnnxOfflineModelConfig(config, Module) {
const fireRedAsr =
initSherpaOnnxOfflineFireRedAsrModelConfig(config.fireRedAsr, Module);
const dolphin =
initSherpaOnnxOfflineDolphinModelConfig(config.dolphin, Module);
const len = transducer.len + paraformer.len + nemoCtc.len + whisper.len +
tdnn.len + 8 * 4 + senseVoice.len + moonshine.len + fireRedAsr.len;
tdnn.len + 8 * 4 + senseVoice.len + moonshine.len + fireRedAsr.len +
dolphin.len;
const ptr = Module._malloc(len);
@@ -932,10 +963,14 @@ function initSherpaOnnxOfflineModelConfig(config, Module) {
Module._CopyHeap(fireRedAsr.ptr, fireRedAsr.len, ptr + offset);
offset += fireRedAsr.len;
Module._CopyHeap(dolphin.ptr, dolphin.len, ptr + offset);
offset += dolphin.len;
return {
buffer: buffer, ptr: ptr, len: len, transducer: transducer,
paraformer: paraformer, nemoCtc: nemoCtc, whisper: whisper, tdnn: tdnn,
senseVoice: senseVoice, moonshine: moonshine, fireRedAsr: fireRedAsr
senseVoice: senseVoice, moonshine: moonshine, fireRedAsr: fireRedAsr,
dolphin: dolphin
}
}

8
wasm/nodejs/sherpa-onnx-wasm-nodejs.cc
View File

@@ -13,6 +13,7 @@ extern "C" {
static_assert(sizeof(SherpaOnnxOfflineTransducerModelConfig) == 3 * 4, "");
static_assert(sizeof(SherpaOnnxOfflineParaformerModelConfig) == 4, "");
static_assert(sizeof(SherpaOnnxOfflineDolphinModelConfig) == 4, "");
static_assert(sizeof(SherpaOnnxOfflineNemoEncDecCtcModelConfig) == 4, "");
static_assert(sizeof(SherpaOnnxOfflineWhisperModelConfig) == 5 * 4, "");
static_assert(sizeof(SherpaOnnxOfflineFireRedAsrModelConfig) == 2 * 4, "");
@@ -29,7 +30,8 @@ static_assert(sizeof(SherpaOnnxOfflineModelConfig) ==
sizeof(SherpaOnnxOfflineTdnnModelConfig) + 8 * 4 +
sizeof(SherpaOnnxOfflineSenseVoiceModelConfig) +
sizeof(SherpaOnnxOfflineMoonshineModelConfig) +
sizeof(SherpaOnnxOfflineFireRedAsrModelConfig),
sizeof(SherpaOnnxOfflineFireRedAsrModelConfig) +
sizeof(SherpaOnnxOfflineDolphinModelConfig),
"");
static_assert(sizeof(SherpaOnnxFeatureConfig) == 2 * 4, "");
@@ -73,6 +75,7 @@ void PrintOfflineRecognizerConfig(SherpaOnnxOfflineRecognizerConfig *config) {
auto sense_voice = &model_config->sense_voice;
auto moonshine = &model_config->moonshine;
auto fire_red_asr = &model_config->fire_red_asr;
auto dolphin = &model_config->dolphin;
fprintf(stdout, "----------offline transducer model config----------\n");
fprintf(stdout, "encoder: %s\n", transducer->encoder);
@@ -110,6 +113,9 @@ void PrintOfflineRecognizerConfig(SherpaOnnxOfflineRecognizerConfig *config) {
fprintf(stdout, "encoder: %s\n", fire_red_asr->encoder);
fprintf(stdout, "decoder: %s\n", fire_red_asr->decoder);
fprintf(stdout, "----------offline Dolphin model config----------\n");
fprintf(stdout, "model: %s\n", dolphin->model);
fprintf(stdout, "tokens: %s\n", model_config->tokens);
fprintf(stdout, "num_threads: %d\n", model_config->num_threads);
fprintf(stdout, "provider: %s\n", model_config->provider);

91
wasm/vad-asr/app-vad-asr.js
View File

@@ -15,7 +15,7 @@ let resultList = [];
clearBtn.onclick = function() {
resultList = [];
textArea.value = getDisplayResult();
textArea.scrollTop = textArea.scrollHeight; // auto scroll
textArea.scrollTop = textArea.scrollHeight; // auto scroll
};
function getDisplayResult() {
@@ -46,11 +46,11 @@ let audioCtx;
let mediaStream;
let expectedSampleRate = 16000;
let recordSampleRate; // the sampleRate of the microphone
let recorder = null; // the microphone
let leftchannel = []; // TODO: Use a single channel
let recordSampleRate; // the sampleRate of the microphone
let recorder = null; // the microphone
let leftchannel = []; // TODO: Use a single channel
let recordingLength = 0; // number of samples so far
let recordingLength = 0; // number of samples so far
let vad = null;
let buffer = null;
@@ -73,48 +73,50 @@ function createOfflineRecognizerSenseVoice() {}
function initOfflineRecognizer() {
let config = {
modelConfig : {
debug : 1,
tokens : './tokens.txt',
modelConfig: {
debug: 1,
tokens: './tokens.txt',
},
};
if (fileExists('sense-voice.onnx') == 1) {
config.modelConfig.senseVoice = {
model : './sense-voice.onnx',
useInverseTextNormalization : 1,
model: './sense-voice.onnx',
useInverseTextNormalization: 1,
};
} else if (fileExists('whisper-encoder.onnx')) {
config.modelConfig.whisper = {
encoder : './whisper-encoder.onnx',
decoder : './whisper-decoder.onnx',
encoder: './whisper-encoder.onnx',
decoder: './whisper-decoder.onnx',
};
} else if (fileExists('transducer-encoder.onnx')) {
config.modelConfig.transducer = {
encoder : './transducer-encoder.onnx',
decoder : './transducer-decoder.onnx',
joiner : './transducer-joiner.onnx',
encoder: './transducer-encoder.onnx',
decoder: './transducer-decoder.onnx',
joiner: './transducer-joiner.onnx',
};
config.modelConfig.modelType = 'transducer';
} else if (fileExists('nemo-transducer-encoder.onnx')) {
config.modelConfig.transducer = {
encoder : './nemo-transducer-encoder.onnx',
decoder : './nemo-transducer-decoder.onnx',
joiner : './nemo-transducer-joiner.onnx',
encoder: './nemo-transducer-encoder.onnx',
decoder: './nemo-transducer-decoder.onnx',
joiner: './nemo-transducer-joiner.onnx',
};
config.modelConfig.modelType = 'nemo_transducer';
} else if (fileExists('paraformer.onnx')) {
config.modelConfig.paraformer = {
model : './paraformer.onnx',
model: './paraformer.onnx',
};
} else if (fileExists('telespeech.onnx')) {
config.modelConfig.telespeechCtc = './telespeech.onnx';
} else if (fileExists('moonshine-preprocessor.onnx')) {
config.modelConfig.moonshine = {
preprocessor : './moonshine-preprocessor.onnx',
encoder : './moonshine-encoder.onnx',
uncachedDecoder : './moonshine-uncached-decoder.onnx',
cachedDecoder : './moonshine-cached-decoder.onnx'
preprocessor: './moonshine-preprocessor.onnx',
encoder: './moonshine-encoder.onnx',
uncachedDecoder: './moonshine-uncached-decoder.onnx',
cachedDecoder: './moonshine-cached-decoder.onnx'
};
} else if (fileExists('dolphin.onnx')) {
config.modelConfig.dolphin = {model: './dolphin.onnx'};
} else {
console.log('Please specify a model.');
alert('Please specify a model.');
@@ -133,7 +135,7 @@ Module.locateFile = function(path, scriptDirectory = '') {
Module.setStatus = function(status) {
console.log(`status ${status}`);
const statusElement = document.getElementById('status');
if (status == "Running...") {
if (status == 'Running...') {
status = 'Model downloaded. Initializing recongizer...'
}
statusElement.textContent = status;
@@ -170,11 +172,11 @@ if (navigator.mediaDevices.getUserMedia) {
console.log('getUserMedia supported.');
// see https://w3c.github.io/mediacapture-main/#dom-mediadevices-getusermedia
const constraints = {audio : true};
const constraints = {audio: true};
let onSuccess = function(stream) {
if (!audioCtx) {
audioCtx = new AudioContext({sampleRate : expectedSampleRate});
audioCtx = new AudioContext({sampleRate: expectedSampleRate});
}
console.log(audioCtx);
recordSampleRate = audioCtx.sampleRate;
@@ -299,7 +301,7 @@ if (navigator.mediaDevices.getUserMedia) {
}
textArea.value = getDisplayResult();
textArea.scrollTop = textArea.scrollHeight; // auto scroll
textArea.scrollTop = textArea.scrollHeight; // auto scroll
};
startBtn.onclick = function() {
@@ -330,8 +332,9 @@ if (navigator.mediaDevices.getUserMedia) {
};
};
let onError = function(
err) { console.log('The following error occured: ' + err); };
let onError = function(err) {
console.log('The following error occured: ' + err);
};
navigator.mediaDevices.getUserMedia(constraints).then(onSuccess, onError);
} else {
@@ -364,22 +367,22 @@ function toWav(samples) {
// http://soundfile.sapp.org/doc/WaveFormat/
// F F I R
view.setUint32(0, 0x46464952, true); // chunkID
view.setUint32(4, 36 + samples.length * 2, true); // chunkSize
view.setUint32(0, 0x46464952, true); // chunkID
view.setUint32(4, 36 + samples.length * 2, true); // chunkSize
// E V A W
view.setUint32(8, 0x45564157, true); // format
//
view.setUint32(8, 0x45564157, true); // format
//
// t m f
view.setUint32(12, 0x20746d66, true); // subchunk1ID
view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM
view.setUint32(20, 1, true); // audioFormat, 1 for PCM
view.setUint16(22, 1, true); // numChannels: 1 channel
view.setUint32(24, expectedSampleRate, true); // sampleRate
view.setUint32(28, expectedSampleRate * 2, true); // byteRate
view.setUint16(32, 2, true); // blockAlign
view.setUint16(34, 16, true); // bitsPerSample
view.setUint32(36, 0x61746164, true); // Subchunk2ID
view.setUint32(40, samples.length * 2, true); // subchunk2Size
view.setUint32(12, 0x20746d66, true); // subchunk1ID
view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM
view.setUint32(20, 1, true); // audioFormat, 1 for PCM
view.setUint16(22, 1, true); // numChannels: 1 channel
view.setUint32(24, expectedSampleRate, true); // sampleRate
view.setUint32(28, expectedSampleRate * 2, true); // byteRate
view.setUint16(32, 2, true); // blockAlign
view.setUint16(34, 16, true); // bitsPerSample
view.setUint32(36, 0x61746164, true); // Subchunk2ID
view.setUint32(40, samples.length * 2, true); // subchunk2Size
let offset = 44;
for (let i = 0; i < samples.length; ++i) {
@@ -387,7 +390,7 @@ function toWav(samples) {
offset += 2;
}
return new Blob([ view ], {type : 'audio/wav'});
return new Blob([view], {type: 'audio/wav'});
}
// this function is copied from