EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix keyword spotting. (#1689)

Browse Source 
Reset the stream right after detecting a keyword
This commit is contained in:
Fangjun Kuang
2025-01-20 16:41:10 +08:00
committed by GitHub
parent b943341fb1
commit 8b989a851c
43 changed files with 813 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
sherpa-onnx/c-api/c-api.cc
View File

@@ -678,7 +678,7 @@ struct SherpaOnnxKeywordSpotter {
std::unique_ptr<sherpa_onnx::KeywordSpotter> impl;
};
SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
const SherpaOnnxKeywordSpotterConfig *config) {
sherpa_onnx::KeywordSpotterConfig spotter_config;
@@ -755,37 +755,42 @@ SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
return spotter;
}
void SherpaOnnxDestroyKeywordSpotter(SherpaOnnxKeywordSpotter *spotter) {
void SherpaOnnxDestroyKeywordSpotter(const SherpaOnnxKeywordSpotter *spotter) {
delete spotter;
}
SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
const SherpaOnnxKeywordSpotter *spotter) {
SherpaOnnxOnlineStream *stream =
new SherpaOnnxOnlineStream(spotter->impl->CreateStream());
return stream;
}
SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords(
const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords(
const SherpaOnnxKeywordSpotter *spotter, const char *keywords) {
SherpaOnnxOnlineStream *stream =
new SherpaOnnxOnlineStream(spotter->impl->CreateStream(keywords));
return stream;
}
int32_t SherpaOnnxIsKeywordStreamReady(SherpaOnnxKeywordSpotter *spotter,
SherpaOnnxOnlineStream *stream) {
int32_t SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream) {
return spotter->impl->IsReady(stream->impl.get());
}
void SherpaOnnxDecodeKeywordStream(SherpaOnnxKeywordSpotter *spotter,
SherpaOnnxOnlineStream *stream) {
return spotter->impl->DecodeStream(stream->impl.get());
void SherpaOnnxDecodeKeywordStream(const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream) {
spotter->impl->DecodeStream(stream->impl.get());
}
void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter,
SherpaOnnxOnlineStream **streams,
int32_t n) {
void SherpaOnnxResetKeywordStream(const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream) {
spotter->impl->Reset(stream->impl.get());
}
void SherpaOnnxDecodeMultipleKeywordStreams(
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream **streams, int32_t n) {
std::vector<sherpa_onnx::OnlineStream *> ss(n);
for (int32_t i = 0; i != n; ++i) {
ss[i] = streams[i]->impl.get();
@@ -794,7 +799,8 @@ void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter,
}
const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream) {
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream) {
const sherpa_onnx::KeywordResult &result =
spotter->impl->GetResult(stream->impl.get());
const auto &keyword = result.keyword;
@@ -869,8 +875,9 @@ void SherpaOnnxDestroyKeywordResult(const SherpaOnnxKeywordResult *r) {
}
}
const char *SherpaOnnxGetKeywordResultAsJson(SherpaOnnxKeywordSpotter *spotter,
SherpaOnnxOnlineStream *stream) {
const char *SherpaOnnxGetKeywordResultAsJson(
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream) {
const sherpa_onnx::KeywordResult &result =
spotter->impl->GetResult(stream->impl.get());

33
sherpa-onnx/c-api/c-api.h
View File

@@ -600,7 +600,7 @@ SHERPA_ONNX_API const char *SherpaOnnxGetOfflineStreamResultAsJson(
SHERPA_ONNX_API void SherpaOnnxDestroyOfflineStreamResultJson(const char *s);
// ============================================================
// For Keyword Spot
// For Keyword Spotter
// ============================================================
SHERPA_ONNX_API typedef struct SherpaOnnxKeywordResult {
/// The triggered keyword.
@@ -660,21 +660,21 @@ SHERPA_ONNX_API typedef struct SherpaOnnxKeywordSpotter
/// @param config Config for the keyword spotter.
/// @return Return a pointer to the spotter. The user has to invoke
/// SherpaOnnxDestroyKeywordSpotter() to free it to avoid memory leak.
SHERPA_ONNX_API SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
SHERPA_ONNX_API const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
const SherpaOnnxKeywordSpotterConfig *config);
/// Free a pointer returned by SherpaOnnxCreateKeywordSpotter()
///
/// @param p A pointer returned by SherpaOnnxCreateKeywordSpotter()
SHERPA_ONNX_API void SherpaOnnxDestroyKeywordSpotter(
SherpaOnnxKeywordSpotter *spotter);
const SherpaOnnxKeywordSpotter *spotter);
/// Create an online stream for accepting wave samples.
///
/// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter()
/// @return Return a pointer to an OnlineStream. The user has to invoke
/// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
SHERPA_ONNX_API const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
const SherpaOnnxKeywordSpotter *spotter);
/// Create an online stream for accepting wave samples with the specified hot
@@ -684,7 +684,7 @@ SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
/// @param keywords A pointer points to the keywords that you set
/// @return Return a pointer to an OnlineStream. The user has to invoke
/// SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
SHERPA_ONNX_API SherpaOnnxOnlineStream *
SHERPA_ONNX_API const SherpaOnnxOnlineStream *
SherpaOnnxCreateKeywordStreamWithKeywords(
const SherpaOnnxKeywordSpotter *spotter, const char *keywords);
@@ -693,15 +693,22 @@ SherpaOnnxCreateKeywordStreamWithKeywords(
///
/// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter
/// @param stream A pointer returned by SherpaOnnxCreateKeywordStream
SHERPA_ONNX_API int32_t SherpaOnnxIsKeywordStreamReady(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
SHERPA_ONNX_API int32_t
SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream);
/// Call this function to run the neural network model and decoding.
//
/// Precondition for this function: SherpaOnnxIsKeywordStreamReady() MUST
/// return 1.
SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream);
/// Please call it right after a keyword is detected
SHERPA_ONNX_API void SherpaOnnxResetKeywordStream(
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream);
/// This function is similar to SherpaOnnxDecodeKeywordStream(). It decodes
/// multiple OnlineStream in parallel.
@@ -714,8 +721,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream(
/// SherpaOnnxCreateKeywordStream()
/// @param n Number of elements in the given streams array.
SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream **streams,
int32_t n);
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream **streams, int32_t n);
/// Get the decoding results so far for an OnlineStream.
///
@@ -725,7 +732,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams(
/// SherpaOnnxDestroyKeywordResult() to free the returned pointer to
/// avoid memory leak.
SHERPA_ONNX_API const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream);
/// Destroy the pointer returned by SherpaOnnxGetKeywordResult().
///
@@ -736,7 +744,8 @@ SHERPA_ONNX_API void SherpaOnnxDestroyKeywordResult(
// the user has to call SherpaOnnxFreeKeywordResultJson() to free the returned
// pointer to avoid memory leak
SHERPA_ONNX_API const char *SherpaOnnxGetKeywordResultAsJson(
SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
const SherpaOnnxKeywordSpotter *spotter,
const SherpaOnnxOnlineStream *stream);
SHERPA_ONNX_API void SherpaOnnxFreeKeywordResultJson(const char *s);

108
sherpa-onnx/c-api/cxx-api.cc
View File

@@ -391,4 +391,112 @@ GeneratedAudio OfflineTts::Generate(const std::string &text,
return ans;
}
KeywordSpotter KeywordSpotter::Create(const KeywordSpotterConfig &config) {
struct SherpaOnnxKeywordSpotterConfig c;
memset(&c, 0, sizeof(c));
c.feat_config.sample_rate = config.feat_config.sample_rate;
c.model_config.transducer.encoder =
config.model_config.transducer.encoder.c_str();
c.model_config.transducer.decoder =
config.model_config.transducer.decoder.c_str();
c.model_config.transducer.joiner =
config.model_config.transducer.joiner.c_str();
c.feat_config.feature_dim = config.feat_config.feature_dim;
c.model_config.paraformer.encoder =
config.model_config.paraformer.encoder.c_str();
c.model_config.paraformer.decoder =
config.model_config.paraformer.decoder.c_str();
c.model_config.zipformer2_ctc.model =
config.model_config.zipformer2_ctc.model.c_str();
c.model_config.tokens = config.model_config.tokens.c_str();
c.model_config.num_threads = config.model_config.num_threads;
c.model_config.provider = config.model_config.provider.c_str();
c.model_config.debug = config.model_config.debug;
c.model_config.model_type = config.model_config.model_type.c_str();
c.model_config.modeling_unit = config.model_config.modeling_unit.c_str();
c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str();
c.model_config.tokens_buf = config.model_config.tokens_buf.c_str();
c.model_config.tokens_buf_size = config.model_config.tokens_buf.size();
c.max_active_paths = config.max_active_paths;
c.num_trailing_blanks = config.num_trailing_blanks;
c.keywords_score = config.keywords_score;
c.keywords_threshold = config.keywords_threshold;
c.keywords_file = config.keywords_file.c_str();
auto p = SherpaOnnxCreateKeywordSpotter(&c);
return KeywordSpotter(p);
}
KeywordSpotter::KeywordSpotter(const SherpaOnnxKeywordSpotter *p)
: MoveOnly<KeywordSpotter, SherpaOnnxKeywordSpotter>(p) {}
void KeywordSpotter::Destroy(const SherpaOnnxKeywordSpotter *p) const {
SherpaOnnxDestroyKeywordSpotter(p);
}
OnlineStream KeywordSpotter::CreateStream() const {
auto s = SherpaOnnxCreateKeywordStream(p_);
return OnlineStream{s};
}
OnlineStream KeywordSpotter::CreateStream(const std::string &keywords) const {
auto s = SherpaOnnxCreateKeywordStreamWithKeywords(p_, keywords.c_str());
return OnlineStream{s};
}
bool KeywordSpotter::IsReady(const OnlineStream *s) const {
return SherpaOnnxIsKeywordStreamReady(p_, s->Get());
}
void KeywordSpotter::Decode(const OnlineStream *s) const {
return SherpaOnnxDecodeKeywordStream(p_, s->Get());
}
void KeywordSpotter::Decode(const OnlineStream *ss, int32_t n) const {
if (n <= 0) {
return;
}
std::vector<const SherpaOnnxOnlineStream *> streams(n);
for (int32_t i = 0; i != n; ++n) {
streams[i] = ss[i].Get();
}
SherpaOnnxDecodeMultipleKeywordStreams(p_, streams.data(), n);
}
KeywordResult KeywordSpotter::GetResult(const OnlineStream *s) const {
auto r = SherpaOnnxGetKeywordResult(p_, s->Get());
KeywordResult ans;
ans.keyword = r->keyword;
ans.tokens.resize(r->count);
for (int32_t i = 0; i < r->count; ++i) {
ans.tokens[i] = r->tokens_arr[i];
}
if (r->timestamps) {
ans.timestamps.resize(r->count);
std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data());
}
ans.start_time = r->start_time;
ans.json = r->json;
SherpaOnnxDestroyKeywordResult(r);
return ans;
}
void KeywordSpotter::Reset(const OnlineStream *s) const {
SherpaOnnxResetKeywordStream(p_, s->Get());
}
} // namespace sherpa_onnx::cxx

47
sherpa-onnx/c-api/cxx-api.h
View File

@@ -406,6 +406,53 @@ class SHERPA_ONNX_API OfflineTts
explicit OfflineTts(const SherpaOnnxOfflineTts *p);
};
// ============================================================
// For Keyword Spotter
// ============================================================
struct KeywordResult {
std::string keyword;
std::vector<std::string> tokens;
std::vector<float> timestamps;
float start_time;
std::string json;
};
struct KeywordSpotterConfig {
FeatureConfig feat_config;
OnlineModelConfig model_config;
int32_t max_active_paths = 4;
int32_t num_trailing_blanks = 1;
float keywords_score = 1.0f;
float keywords_threshold = 0.25f;
std::string keywords_file;
};
class SHERPA_ONNX_API KeywordSpotter
: public MoveOnly<KeywordSpotter, SherpaOnnxKeywordSpotter> {
public:
static KeywordSpotter Create(const KeywordSpotterConfig &config);
void Destroy(const SherpaOnnxKeywordSpotter *p) const;
OnlineStream CreateStream() const;
OnlineStream CreateStream(const std::string &keywords) const;
bool IsReady(const OnlineStream *s) const;
void Decode(const OnlineStream *s) const;
void Decode(const OnlineStream *ss, int32_t n) const;
void Reset(const OnlineStream *s) const;
KeywordResult GetResult(const OnlineStream *s) const;
private:
explicit KeywordSpotter(const SherpaOnnxKeywordSpotter *p);
};
} // namespace sherpa_onnx::cxx
#endif // SHERPA_ONNX_C_API_CXX_API_H_

2
sherpa-onnx/csrc/keyword-spotter-impl.h
View File

@@ -38,6 +38,8 @@ class KeywordSpotterImpl {
virtual bool IsReady(OnlineStream *s) const = 0;
virtual void Reset(OnlineStream *s) const = 0;
virtual void DecodeStreams(OnlineStream **ss, int32_t n) const = 0;
virtual KeywordResult GetResult(OnlineStream *s) const = 0;

16
sherpa-onnx/csrc/keyword-spotter-transducer-impl.h
View File

@@ -195,8 +195,24 @@ class KeywordSpotterTransducerImpl : public KeywordSpotterImpl {
return s->GetNumProcessedFrames() + model_->ChunkSize() <
s->NumFramesReady();
}
void Reset(OnlineStream *s) const override { InitOnlineStream(s); }
void DecodeStreams(OnlineStream **ss, int32_t n) const override {
for (int32_t i = 0; i < n; ++i) {
auto s = ss[i];
auto r = s->GetKeywordResult(true);
int32_t num_trailing_blanks = r.num_trailing_blanks;
// assume subsampling_factor is 4
// assume frameshift is 0.01 second
float trailing_slience = num_trailing_blanks * 4 * 0.01;
// it resets automatically after detecting 1.5 seconds of silence
float threshold = 1.5;
if (trailing_slience > threshold) {
Reset(s);
}
}
int32_t chunk_size = model_->ChunkSize();
int32_t chunk_shift = model_->ChunkShift();

2
sherpa-onnx/csrc/keyword-spotter.cc
View File

@@ -157,6 +157,8 @@ bool KeywordSpotter::IsReady(OnlineStream *s) const {
return impl_->IsReady(s);
}
void KeywordSpotter::Reset(OnlineStream *s) const { impl_->Reset(s); }
void KeywordSpotter::DecodeStreams(OnlineStream **ss, int32_t n) const {
impl_->DecodeStreams(ss, n);
}

3
sherpa-onnx/csrc/keyword-spotter.h
View File

@@ -129,6 +129,9 @@ class KeywordSpotter {
*/
bool IsReady(OnlineStream *s) const;
// Remember to call it after detecting a keyword
void Reset(OnlineStream *s) const;
/** Decode a single stream. */
void DecodeStream(OnlineStream *s) const {
OnlineStream *ss[1] = {s};

14
sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc
View File

@@ -106,13 +106,15 @@ as the device_name.
while (spotter.IsReady(stream.get())) {
spotter.DecodeStream(stream.get());
}
const auto r = spotter.GetResult(stream.get());
if (!r.keyword.empty()) {
display.Print(keyword_index, r.AsJsonString());
fflush(stderr);
keyword_index++;
const auto r = spotter.GetResult(stream.get());
if (!r.keyword.empty()) {
display.Print(keyword_index, r.AsJsonString());
fflush(stderr);
keyword_index++;
spotter.Reset(stream.get());
}
}
}

14
sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc
View File

@@ -150,13 +150,15 @@ for a list of pre-trained models to download.
while (!stop) {
while (spotter.IsReady(s.get())) {
spotter.DecodeStream(s.get());
}
const auto r = spotter.GetResult(s.get());
if (!r.keyword.empty()) {
display.Print(keyword_index, r.AsJsonString());
fflush(stderr);
keyword_index++;
const auto r = spotter.GetResult(s.get());
if (!r.keyword.empty()) {
display.Print(keyword_index, r.AsJsonString());
fflush(stderr);
keyword_index++;
spotter.Reset(s.get());
}
}
Pa_Sleep(20); // sleep for 20ms

6
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/KeywordSpotter.java
View File

@@ -27,6 +27,10 @@ public class KeywordSpotter {
decode(ptr, s.getPtr());
}
public void reset(OnlineStream s) {
reset(ptr, s.getPtr());
}
public boolean isReady(OnlineStream s) {
return isReady(ptr, s.getPtr());
}
@@ -60,6 +64,8 @@ public class KeywordSpotter {
private native void decode(long ptr, long streamPtr);
private native void reset(long ptr, long streamPtr);
private native boolean isReady(long ptr, long streamPtr);
private native Object[] getResult(long ptr, long streamPtr);

9
sherpa-onnx/jni/keyword-spotter.cc
View File

@@ -161,6 +161,15 @@ JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_decode(
kws->DecodeStream(stream);
}
SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_reset(
JNIEnv * /*env*/, jobject /*obj*/, jlong ptr, jlong stream_ptr) {
auto kws = reinterpret_cast<sherpa_onnx::KeywordSpotter *>(ptr);
auto stream = reinterpret_cast<sherpa_onnx::OnlineStream *>(stream_ptr);
kws->Reset(stream);
}
SHERPA_ONNX_EXTERN_C
JNIEXPORT jlong JNICALL Java_com_k2fsa_sherpa_onnx_KeywordSpotter_createStream(
JNIEnv *env, jobject /*obj*/, jlong ptr, jstring keywords) {

2
sherpa-onnx/kotlin-api/KeywordSpotter.kt
View File

@@ -49,6 +49,7 @@ class KeywordSpotter(
}
fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
fun getResult(stream: OnlineStream): KeywordSpotterResult {
val objArray = getResult(ptr, stream.ptr)
@@ -74,6 +75,7 @@ class KeywordSpotter(
private external fun createStream(ptr: Long, keywords: String): Long
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
private external fun decode(ptr: Long, streamPtr: Long)
private external fun reset(ptr: Long, streamPtr: Long)
private external fun getResult(ptr: Long, streamPtr: Long): Array<Any>
companion object {

1
sherpa-onnx/python/csrc/keyword-spotter.cc
View File

@@ -67,6 +67,7 @@ void PybindKeywordSpotter(py::module *m) {
py::arg("keywords"), py::call_guard<py::gil_scoped_release>())
.def("is_ready", &PyClass::IsReady,
py::call_guard<py::gil_scoped_release>())
.def("reset", &PyClass::Reset, py::call_guard<py::gil_scoped_release>())
.def("decode_stream", &PyClass::DecodeStream,
py::call_guard<py::gil_scoped_release>())
.def(

7
sherpa-onnx/python/sherpa_onnx/keyword_spotter.py
View File

@@ -104,8 +104,8 @@ class KeywordSpotter(object):
)
provider_config = ProviderConfig(
provider=provider,
device = device,
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
@@ -131,6 +131,9 @@ class KeywordSpotter(object):
)
self.keyword_spotter = _KeywordSpotter(keywords_spotter_config)
def reset_stream(self, s: OnlineStream):
self.keyword_spotter.reset(s)
def create_stream(self, keywords: Optional[str] = None):
if keywords is None:
return self.keyword_spotter.create_stream()

6
sherpa-onnx/python/tests/test_keyword_spotter.py
View File

@@ -98,6 +98,9 @@ class TestKeywordSpotter(unittest.TestCase):
if r:
print(f"{r} is detected.")
results[i] += f"{r}/"
keyword_spotter.reset_stream(s)
if len(ready_list) == 0:
break
keyword_spotter.decode_streams(ready_list)
@@ -158,6 +161,9 @@ class TestKeywordSpotter(unittest.TestCase):
if r:
print(f"{r} is detected.")
results[i] += f"{r}/"
keyword_spotter.reset_stream(s)
if len(ready_list) == 0:
break
keyword_spotter.decode_streams(ready_list)