EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
15
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add keyword spotting API for node-addon-api (#877)

Browse Source
This commit is contained in:
Fangjun Kuang
2024-05-14 20:26:48 +08:00
committed by GitHub
parent 75630b986b
commit 03c956a317
18 changed files with 492 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/scripts/node-addon/run.sh vendored
View File

@@ -18,7 +18,7 @@ fi
SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2)
echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
# SHERPA_ONNX_VERSION=1.0.23
# SHERPA_ONNX_VERSION=1.0.24
if [ -z $owner ]; then
owner=k2-fsa

9
.github/scripts/test-nodejs-addon-npm.sh vendored
View File

@@ -6,6 +6,15 @@ d=nodejs-addon-examples
echo "dir: $d"
cd $d
echo "----------keyword spotting----------"
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
node ./test_keyword_spotter_transducer.js
rm -rf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01
echo "----------add punctuations----------"
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2

2
.github/workflows/npm-addon.yaml vendored
View File

@@ -55,7 +55,7 @@ jobs:
SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2)
echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION"
# SHERPA_ONNX_VERSION=1.0.23
# SHERPA_ONNX_VERSION=1.0.24
src_dir=.github/scripts/node-addon
sed -i.bak s/SHERPA_ONNX_VERSION/$SHERPA_ONNX_VERSION/g $src_dir/package.json

21
nodejs-addon-examples/README.md
View File

@@ -62,6 +62,13 @@ The following tables list the examples in this folder.
|[./test_audio_tagging_zipformer.js](./test_audio_tagging_zipformer.js)| Audio tagging with a Zipformer model|
|[./test_audio_tagging_ced.js](./test_audio_tagging_ced.js)| Audio tagging with a [CED](https://github.com/RicherMans/CED) model|
## Keyword spotting
|File| Description|
|---|---|
|[./test_keyword_spotter_transducer.js](./test_keyword_spotter_transducer.js)| Keyword spotting from a file using a Zipformer model|
|[./test_keyword_spotter_transducer_microphone.js](./test_keyword_spotter_transducer_microphone.js)| Keyword spotting from a microphone using a Zipformer model|
## Streaming speech-to-text from files
|File| Description|
@@ -325,3 +332,17 @@ rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
node ./test_punctuation.js
```
## Keyword spotting
```bash
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
tar xvf sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
rm sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01.tar.bz2
node ./test_keyword_spotter_transducer.js
# To run keyword spotting using a microphone
npm install naudiodon2
node ./test_keyword_spotter_transducer_microphone.js
```

6
nodejs-addon-examples/test_asr_streaming_ctc_hlg_microphone.js
View File

@@ -79,11 +79,5 @@ ai.on('data', data => {
}
});
ai.on('close', () => {
console.log('Free resources');
stream.free();
recognizer.free();
});
ai.start();
console.log('Started! Please speak')

5
nodejs-addon-examples/test_asr_streaming_ctc_microphone.js
View File

@@ -78,11 +78,6 @@ ai.on('data', data => {
}
});
ai.on('close', () => {
console.log('Free resources');
stream.free();
recognizer.free();
});
ai.start();
console.log('Started! Please speak')

6
nodejs-addon-examples/test_asr_streaming_paraformer_microphone.js
View File

@@ -94,11 +94,5 @@ ai.on('data', data => {
}
});
ai.on('close', () => {
console.log('Free resources');
stream.free();
recognizer.free();
});
ai.start();
console.log('Started! Please speak')

6
nodejs-addon-examples/test_asr_streaming_transducer_microphone.js
View File

@@ -82,11 +82,5 @@ ai.on('data', data => {
}
});
ai.on('close', () => {
console.log('Free resources');
stream.free();
recognizer.free();
});
ai.start();
console.log('Started! Please speak')

66
nodejs-addon-examples/test_keyword_spotter_transducer.js Normal file
View File

@@ -0,0 +1,66 @@
// Copyright (c) 2024 Xiaomi Corporation
const sherpa_onnx = require('sherpa-onnx-node');
const performance = require('perf_hooks').performance;
// Please download test files from
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/kws-models
const config = {
'featConfig': {
'sampleRate': 16000,
'featureDim': 80,
},
'modelConfig': {
'transducer': {
'encoder':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx',
'decoder':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx',
'joiner':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx',
},
'tokens':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt',
'numThreads': 1,
'provider': 'cpu',
'debug': 1,
},
'keywordsFile':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/test_keywords.txt',
};
const waveFilename =
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav';
const kws = new sherpa_onnx.KeywordSpotter(config);
console.log('Started')
let start = performance.now();
const stream = kws.createStream();
const wave = sherpa_onnx.readWave(waveFilename);
stream.acceptWaveform({sampleRate: wave.sampleRate, samples: wave.samples});
const tailPadding = new Float32Array(wave.sampleRate * 0.4);
stream.acceptWaveform({samples: tailPadding, sampleRate: wave.sampleRate});
const detectedKeywords = [];
while (kws.isReady(stream)) {
const keyword = kws.getResult(stream).keyword;
if (keyword != '') {
detectedKeywords.push(keyword);
}
kws.decode(stream);
}
let stop = performance.now();
console.log('Done')
const elapsed_seconds = (stop - start) / 1000;
const duration = wave.samples.length / wave.sampleRate;
const real_time_factor = elapsed_seconds / duration;
console.log('Wave duration', duration.toFixed(3), 'secodns')
console.log('Elapsed', elapsed_seconds.toFixed(3), 'secodns')
console.log(
`RTF = ${elapsed_seconds.toFixed(3)}/${duration.toFixed(3)} =`,
real_time_factor.toFixed(3))
console.log(waveFilename)
console.log('result\n', detectedKeywords)

74
nodejs-addon-examples/test_keyword_spotter_transducer_microphone.js Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (c) 2023-2024 Xiaomi Corporation (authors: Fangjun Kuang)
//
const portAudio = require('naudiodon2');
// console.log(portAudio.getDevices());
const sherpa_onnx = require('sherpa-onnx-node');
function createKeywordSpotter() {
const config = {
'featConfig': {
'sampleRate': 16000,
'featureDim': 80,
},
'modelConfig': {
'transducer': {
'encoder':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx',
'decoder':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx',
'joiner':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx',
},
'tokens':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt',
'numThreads': 2,
'provider': 'cpu',
'debug': 1,
},
'keywordsFile':
'./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/keywords.txt',
};
return new sherpa_onnx.KeywordSpotter(config);
}
const kws = createKeywordSpotter();
const stream = kws.createStream();
let lastText = '';
let segmentIndex = 0;
const ai = new portAudio.AudioIO({
inOptions: {
channelCount: 1,
closeOnError: true, // Close the stream if an audio error is detected, if
// set false then just log the error
deviceId: -1, // Use -1 or omit the deviceId to select the default device
sampleFormat: portAudio.SampleFormatFloat32,
sampleRate: kws.config.featConfig.sampleRate
}
});
const display = new sherpa_onnx.Display(50);
ai.on('data', data => {
const samples = new Float32Array(data.buffer);
stream.acceptWaveform(
{sampleRate: kws.config.featConfig.sampleRate, samples: samples});
while (kws.isReady(stream)) {
kws.decode(stream);
}
const keyword = kws.getResult(stream).keyword
if (keyword != '') {
display.print(segmentIndex, keyword);
segmentIndex += 1;
}
});
ai.start();
console.log('Started! Please speak.')
console.log(`Only words from ${kws.config.keywordsFile} can be recognized`)

1
scripts/node-addon-api/CMakeLists.txt
View File

@@ -19,6 +19,7 @@ include_directories(${CMAKE_JS_INC})
set(srcs
src/audio-tagging.cc
src/keyword-spotting.cc
src/non-streaming-asr.cc
src/non-streaming-tts.cc
src/punctuation.cc

32
scripts/node-addon-api/lib/keyword-spotter.js Normal file
View File

@@ -0,0 +1,32 @@
const addon = require('./addon.js');
const streaming_asr = require('./streaming-asr.js');
class KeywordSpotter {
constructor(config) {
this.handle = addon.createKeywordSpotter(config);
this.config = config
}
createStream() {
const handle = addon.createKeywordStream(this.handle);
return new streaming_asr.OnlineStream(handle);
}
isReady(stream) {
return addon.isKeywordStreamReady(this.handle, stream.handle);
}
decode(stream) {
addon.decodeKeywordStream(this.handle, stream.handle);
}
getResult(stream) {
const jsonStr = addon.getKeywordResultAsJson(this.handle, stream.handle);
return JSON.parse(jsonStr);
}
}
module.exports = {
KeywordSpotter,
}

2
scripts/node-addon-api/lib/sherpa-onnx.js
View File

@@ -7,6 +7,7 @@ const slid = require('./spoken-language-identification.js');
const sid = require('./speaker-identification.js');
const at = require('./audio-tagg.js');
const punct = require('./punctuation.js');
const kws = require('./keyword-spotter.js');
module.exports = {
OnlineRecognizer: streaming_asr.OnlineRecognizer,
@@ -22,4 +23,5 @@ module.exports = {
SpeakerEmbeddingManager: sid.SpeakerEmbeddingManager,
AudioTagging: at.AudioTagging,
Punctuation: punct.Punctuation,
KeywordSpotter: kws.KeywordSpotter,
}

260
scripts/node-addon-api/src/keyword-spotting.cc Normal file
View File

@@ -0,0 +1,260 @@
// scripts/node-addon-api/src/keyword-spotting.cc
//
// Copyright (c) 2024 Xiaomi Corporation
#include <sstream>
#include "macros.h" // NOLINT
#include "napi.h" // NOLINT
#include "sherpa-onnx/c-api/c-api.h"
// defined ./streaming-asr.cc
SherpaOnnxFeatureConfig GetFeatureConfig(Napi::Object obj);
// defined ./streaming-asr.cc
SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj);
static Napi::External<SherpaOnnxKeywordSpotter> CreateKeywordSpotterWrapper(
const Napi::CallbackInfo &info) {
Napi::Env env = info.Env();
if (info.Length() != 1) {
std::ostringstream os;
os << "Expect only 1 argument. Given: " << info.Length();
Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
return {};
}
if (!info[0].IsObject()) {
Napi::TypeError::New(env, "Expect an object as the argument")
.ThrowAsJavaScriptException();
return {};
}
Napi::Object o = info[0].As<Napi::Object>();
SherpaOnnxKeywordSpotterConfig c;
memset(&c, 0, sizeof(c));
c.feat_config = GetFeatureConfig(o);
c.model_config = GetOnlineModelConfig(o);
SHERPA_ONNX_ASSIGN_ATTR_INT32(max_active_paths, maxActivePaths);
SHERPA_ONNX_ASSIGN_ATTR_INT32(num_trailing_blanks, numTrailingBlanks);
SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_score, keywordsScore);
SHERPA_ONNX_ASSIGN_ATTR_FLOAT(keywords_threshold, keywordsThreshold);
SHERPA_ONNX_ASSIGN_ATTR_STR(keywords_file, keywordsFile);
SherpaOnnxKeywordSpotter *kws = CreateKeywordSpotter(&c);
if (c.model_config.transducer.encoder) {
delete[] c.model_config.transducer.encoder;
}
if (c.model_config.transducer.decoder) {
delete[] c.model_config.transducer.decoder;
}
if (c.model_config.transducer.joiner) {
delete[] c.model_config.transducer.joiner;
}
if (c.model_config.paraformer.encoder) {
delete[] c.model_config.paraformer.encoder;
}
if (c.model_config.paraformer.decoder) {
delete[] c.model_config.paraformer.decoder;
}
if (c.model_config.zipformer2_ctc.model) {
delete[] c.model_config.zipformer2_ctc.model;
}
if (c.model_config.tokens) {
delete[] c.model_config.tokens;
}
if (c.model_config.provider) {
delete[] c.model_config.provider;
}
if (c.model_config.model_type) {
delete[] c.model_config.model_type;
}
if (c.keywords_file) {
delete[] c.keywords_file;
}
if (!kws) {
Napi::TypeError::New(env, "Please check your config!")
.ThrowAsJavaScriptException();
return {};
}
return Napi::External<SherpaOnnxKeywordSpotter>::New(
env, kws, [](Napi::Env env, SherpaOnnxKeywordSpotter *kws) {
DestroyKeywordSpotter(kws);
});
}
static Napi::External<SherpaOnnxOnlineStream> CreateKeywordStreamWrapper(
const Napi::CallbackInfo &info) {
Napi::Env env = info.Env();
if (info.Length() != 1) {
std::ostringstream os;
os << "Expect only 1 argument. Given: " << info.Length();
Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
return {};
}
if (!info[0].IsExternal()) {
Napi::TypeError::New(
env, "You should pass a keyword spotter pointer as the only argument")
.ThrowAsJavaScriptException();
return {};
}
SherpaOnnxKeywordSpotter *kws =
info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
SherpaOnnxOnlineStream *stream = CreateKeywordStream(kws);
return Napi::External<SherpaOnnxOnlineStream>::New(
env, stream, [](Napi::Env env, SherpaOnnxOnlineStream *stream) {
DestroyOnlineStream(stream);
});
}
static Napi::Boolean IsKeywordStreamReadyWrapper(
const Napi::CallbackInfo &info) {
Napi::Env env = info.Env();
if (info.Length() != 2) {
std::ostringstream os;
os << "Expect only 2 arguments. Given: " << info.Length();
Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
return {};
}
if (!info[0].IsExternal()) {
Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
.ThrowAsJavaScriptException();
return {};
}
if (!info[1].IsExternal()) {
Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
.ThrowAsJavaScriptException();
return {};
}
SherpaOnnxKeywordSpotter *kws =
info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
SherpaOnnxOnlineStream *stream =
info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
int32_t is_ready = IsKeywordStreamReady(kws, stream);
return Napi::Boolean::New(env, is_ready);
}
static void DecodeKeywordStreamWrapper(const Napi::CallbackInfo &info) {
Napi::Env env = info.Env();
if (info.Length() != 2) {
std::ostringstream os;
os << "Expect only 2 arguments. Given: " << info.Length();
Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
return;
}
if (!info[0].IsExternal()) {
Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
.ThrowAsJavaScriptException();
return;
}
if (!info[1].IsExternal()) {
Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
.ThrowAsJavaScriptException();
return;
}
SherpaOnnxKeywordSpotter *kws =
info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
SherpaOnnxOnlineStream *stream =
info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
DecodeKeywordStream(kws, stream);
}
static Napi::String GetKeywordResultAsJsonWrapper(
const Napi::CallbackInfo &info) {
Napi::Env env = info.Env();
if (info.Length() != 2) {
std::ostringstream os;
os << "Expect only 2 arguments. Given: " << info.Length();
Napi::TypeError::New(env, os.str()).ThrowAsJavaScriptException();
return {};
}
if (!info[0].IsExternal()) {
Napi::TypeError::New(env, "Argument 0 should be a keyword spotter pointer.")
.ThrowAsJavaScriptException();
return {};
}
if (!info[1].IsExternal()) {
Napi::TypeError::New(env, "Argument 1 should be an online stream pointer.")
.ThrowAsJavaScriptException();
return {};
}
SherpaOnnxKeywordSpotter *kws =
info[0].As<Napi::External<SherpaOnnxKeywordSpotter>>().Data();
SherpaOnnxOnlineStream *stream =
info[1].As<Napi::External<SherpaOnnxOnlineStream>>().Data();
const char *json = GetKeywordResultAsJson(kws, stream);
Napi::String s = Napi::String::New(env, json);
FreeKeywordResultJson(json);
return s;
}
void InitKeywordSpotting(Napi::Env env, Napi::Object exports) {
exports.Set(Napi::String::New(env, "createKeywordSpotter"),
Napi::Function::New(env, CreateKeywordSpotterWrapper));
exports.Set(Napi::String::New(env, "createKeywordStream"),
Napi::Function::New(env, CreateKeywordStreamWrapper));
exports.Set(Napi::String::New(env, "isKeywordStreamReady"),
Napi::Function::New(env, IsKeywordStreamReadyWrapper));
exports.Set(Napi::String::New(env, "decodeKeywordStream"),
Napi::Function::New(env, DecodeKeywordStreamWrapper));
exports.Set(Napi::String::New(env, "getKeywordResultAsJson"),
Napi::Function::New(env, GetKeywordResultAsJsonWrapper));
}

3
scripts/node-addon-api/src/sherpa-onnx-node-addon-api.cc
View File

@@ -23,6 +23,8 @@ void InitAudioTagging(Napi::Env env, Napi::Object exports);
void InitPunctuation(Napi::Env env, Napi::Object exports);
void InitKeywordSpotting(Napi::Env env, Napi::Object exports);
Napi::Object Init(Napi::Env env, Napi::Object exports) {
InitStreamingAsr(env, exports);
InitNonStreamingAsr(env, exports);
@@ -34,6 +36,7 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) {
InitSpeakerID(env, exports);
InitAudioTagging(env, exports);
InitPunctuation(env, exports);
InitKeywordSpotting(env, exports);
return exports;
}

2
scripts/node-addon-api/src/streaming-asr.cc
View File

@@ -90,7 +90,7 @@ static SherpaOnnxOnlineParaformerModelConfig GetOnlineParaformerModelConfig(
return c;
}
static SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) {
SherpaOnnxOnlineModelConfig GetOnlineModelConfig(Napi::Object obj) {
SherpaOnnxOnlineModelConfig c;
memset(&c, 0, sizeof(c));

14
sherpa-onnx/c-api/c-api.cc
View File

@@ -653,6 +653,20 @@ void DestroyKeywordResult(const SherpaOnnxKeywordResult *r) {
}
}
const char *GetKeywordResultAsJson(SherpaOnnxKeywordSpotter *spotter,
SherpaOnnxOnlineStream *stream) {
const sherpa_onnx::KeywordResult &result =
spotter->impl->GetResult(stream->impl.get());
std::string json = result.AsJsonString();
char *pJson = new char[json.size() + 1];
std::copy(json.begin(), json.end(), pJson);
pJson[json.size()] = 0;
return pJson;
}
void FreeKeywordResultJson(const char *s) { delete[] s; }
// ============================================================
// For VAD
// ============================================================

7
sherpa-onnx/c-api/c-api.h
View File

@@ -625,6 +625,13 @@ SHERPA_ONNX_API const SherpaOnnxKeywordResult *GetKeywordResult(
/// @param r A pointer returned by GetKeywordResult()
SHERPA_ONNX_API void DestroyKeywordResult(const SherpaOnnxKeywordResult *r);
// the user has to call FreeKeywordResultJson() to free the returned pointer
// to avoid memory leak
SHERPA_ONNX_API const char *GetKeywordResultAsJson(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
SHERPA_ONNX_API void FreeKeywordResultJson(const char *s);
// ============================================================
// For VAD
// ============================================================