Fix keyword spotting. (#1689)

Reset the stream right after detecting a keyword

sherpa-onnx/c-api/c-api.cc

@@ -678,7 +678,7 @@ struct SherpaOnnxKeywordSpotter {
   std::unique_ptr<sherpa_onnx::KeywordSpotter> impl;
 };
 
-SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
+const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
     const SherpaOnnxKeywordSpotterConfig *config) {
   sherpa_onnx::KeywordSpotterConfig spotter_config;
 
@@ -755,37 +755,42 @@ SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
   return spotter;
 }
 
-void SherpaOnnxDestroyKeywordSpotter(SherpaOnnxKeywordSpotter *spotter) {
+void SherpaOnnxDestroyKeywordSpotter(const SherpaOnnxKeywordSpotter *spotter) {
   delete spotter;
 }
 
-SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
+const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
     const SherpaOnnxKeywordSpotter *spotter) {
   SherpaOnnxOnlineStream *stream =
       new SherpaOnnxOnlineStream(spotter->impl->CreateStream());
   return stream;
 }
 
-SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords(
+const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStreamWithKeywords(
     const SherpaOnnxKeywordSpotter *spotter, const char *keywords) {
   SherpaOnnxOnlineStream *stream =
       new SherpaOnnxOnlineStream(spotter->impl->CreateStream(keywords));
   return stream;
 }
 
-int32_t SherpaOnnxIsKeywordStreamReady(SherpaOnnxKeywordSpotter *spotter,
-                                       SherpaOnnxOnlineStream *stream) {
+int32_t SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter,
+                                       const SherpaOnnxOnlineStream *stream) {
   return spotter->impl->IsReady(stream->impl.get());
 }
 
-void SherpaOnnxDecodeKeywordStream(SherpaOnnxKeywordSpotter *spotter,
-                                   SherpaOnnxOnlineStream *stream) {
-  return spotter->impl->DecodeStream(stream->impl.get());
+void SherpaOnnxDecodeKeywordStream(const SherpaOnnxKeywordSpotter *spotter,
+                                   const SherpaOnnxOnlineStream *stream) {
+  spotter->impl->DecodeStream(stream->impl.get());
 }
 
-void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter,
-                                            SherpaOnnxOnlineStream **streams,
-                                            int32_t n) {
+void SherpaOnnxResetKeywordStream(const SherpaOnnxKeywordSpotter *spotter,
+                                  const SherpaOnnxOnlineStream *stream) {
+  spotter->impl->Reset(stream->impl.get());
+}
+
+void SherpaOnnxDecodeMultipleKeywordStreams(
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream **streams, int32_t n) {
   std::vector<sherpa_onnx::OnlineStream *> ss(n);
   for (int32_t i = 0; i != n; ++i) {
     ss[i] = streams[i]->impl.get();
@@ -794,7 +799,8 @@ void SherpaOnnxDecodeMultipleKeywordStreams(SherpaOnnxKeywordSpotter *spotter,
 }
 
 const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream) {
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream) {
   const sherpa_onnx::KeywordResult &result =
       spotter->impl->GetResult(stream->impl.get());
   const auto &keyword = result.keyword;
@@ -869,8 +875,9 @@ void SherpaOnnxDestroyKeywordResult(const SherpaOnnxKeywordResult *r) {
   }
 }
 
-const char *SherpaOnnxGetKeywordResultAsJson(SherpaOnnxKeywordSpotter *spotter,
-                                             SherpaOnnxOnlineStream *stream) {
+const char *SherpaOnnxGetKeywordResultAsJson(
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream) {
   const sherpa_onnx::KeywordResult &result =
       spotter->impl->GetResult(stream->impl.get());
 

sherpa-onnx/c-api/c-api.h

@@ -600,7 +600,7 @@ SHERPA_ONNX_API const char *SherpaOnnxGetOfflineStreamResultAsJson(
 SHERPA_ONNX_API void SherpaOnnxDestroyOfflineStreamResultJson(const char *s);
 
 // ============================================================
-// For Keyword Spot
+// For Keyword Spotter
 // ============================================================
 SHERPA_ONNX_API typedef struct SherpaOnnxKeywordResult {
   /// The triggered keyword.
@@ -660,21 +660,21 @@ SHERPA_ONNX_API typedef struct SherpaOnnxKeywordSpotter
 /// @param config  Config for the keyword spotter.
 /// @return Return a pointer to the spotter. The user has to invoke
 ///         SherpaOnnxDestroyKeywordSpotter() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
+SHERPA_ONNX_API const SherpaOnnxKeywordSpotter *SherpaOnnxCreateKeywordSpotter(
     const SherpaOnnxKeywordSpotterConfig *config);
 
 /// Free a pointer returned by SherpaOnnxCreateKeywordSpotter()
 ///
 /// @param p A pointer returned by SherpaOnnxCreateKeywordSpotter()
 SHERPA_ONNX_API void SherpaOnnxDestroyKeywordSpotter(
-    SherpaOnnxKeywordSpotter *spotter);
+    const SherpaOnnxKeywordSpotter *spotter);
 
 /// Create an online stream for accepting wave samples.
 ///
 /// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter()
 /// @return Return a pointer to an OnlineStream. The user has to invoke
 ///         SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
+SHERPA_ONNX_API const SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
     const SherpaOnnxKeywordSpotter *spotter);
 
 /// Create an online stream for accepting wave samples with the specified hot
@@ -684,7 +684,7 @@ SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateKeywordStream(
 /// @param keywords A pointer points to the keywords that you set
 /// @return Return a pointer to an OnlineStream. The user has to invoke
 ///         SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxOnlineStream *
+SHERPA_ONNX_API const SherpaOnnxOnlineStream *
 SherpaOnnxCreateKeywordStreamWithKeywords(
     const SherpaOnnxKeywordSpotter *spotter, const char *keywords);
 
@@ -693,15 +693,22 @@ SherpaOnnxCreateKeywordStreamWithKeywords(
 ///
 /// @param spotter A pointer returned by SherpaOnnxCreateKeywordSpotter
 /// @param stream  A pointer returned by SherpaOnnxCreateKeywordStream
-SHERPA_ONNX_API int32_t SherpaOnnxIsKeywordStreamReady(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
+SHERPA_ONNX_API int32_t
+SherpaOnnxIsKeywordStreamReady(const SherpaOnnxKeywordSpotter *spotter,
+                               const SherpaOnnxOnlineStream *stream);
 
 /// Call this function to run the neural network model and decoding.
 //
 /// Precondition for this function: SherpaOnnxIsKeywordStreamReady() MUST
 /// return 1.
 SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream);
+
+/// Please call it right after a keyword is detected
+SHERPA_ONNX_API void SherpaOnnxResetKeywordStream(
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream);
 
 /// This function is similar to SherpaOnnxDecodeKeywordStream(). It decodes
 /// multiple OnlineStream in parallel.
@@ -714,8 +721,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeKeywordStream(
 ///                 SherpaOnnxCreateKeywordStream()
 /// @param n  Number of elements in the given streams array.
 SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream **streams,
-    int32_t n);
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream **streams, int32_t n);
 
 /// Get the decoding results so far for an OnlineStream.
 ///
@@ -725,7 +732,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeMultipleKeywordStreams(
 ///         SherpaOnnxDestroyKeywordResult() to free the returned pointer to
 ///         avoid memory leak.
 SHERPA_ONNX_API const SherpaOnnxKeywordResult *SherpaOnnxGetKeywordResult(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream);
 
 /// Destroy the pointer returned by SherpaOnnxGetKeywordResult().
 ///
@@ -736,7 +744,8 @@ SHERPA_ONNX_API void SherpaOnnxDestroyKeywordResult(
 // the user has to call SherpaOnnxFreeKeywordResultJson() to free the returned
 // pointer to avoid memory leak
 SHERPA_ONNX_API const char *SherpaOnnxGetKeywordResultAsJson(
-    SherpaOnnxKeywordSpotter *spotter, SherpaOnnxOnlineStream *stream);
+    const SherpaOnnxKeywordSpotter *spotter,
+    const SherpaOnnxOnlineStream *stream);
 
 SHERPA_ONNX_API void SherpaOnnxFreeKeywordResultJson(const char *s);
 

sherpa-onnx/c-api/cxx-api.cc

@@ -391,4 +391,112 @@ GeneratedAudio OfflineTts::Generate(const std::string &text,
   return ans;
 }
 
+KeywordSpotter KeywordSpotter::Create(const KeywordSpotterConfig &config) {
+  struct SherpaOnnxKeywordSpotterConfig c;
+  memset(&c, 0, sizeof(c));
+
+  c.feat_config.sample_rate = config.feat_config.sample_rate;
+
+  c.model_config.transducer.encoder =
+      config.model_config.transducer.encoder.c_str();
+  c.model_config.transducer.decoder =
+      config.model_config.transducer.decoder.c_str();
+  c.model_config.transducer.joiner =
+      config.model_config.transducer.joiner.c_str();
+  c.feat_config.feature_dim = config.feat_config.feature_dim;
+
+  c.model_config.paraformer.encoder =
+      config.model_config.paraformer.encoder.c_str();
+  c.model_config.paraformer.decoder =
+      config.model_config.paraformer.decoder.c_str();
+
+  c.model_config.zipformer2_ctc.model =
+      config.model_config.zipformer2_ctc.model.c_str();
+
+  c.model_config.tokens = config.model_config.tokens.c_str();
+  c.model_config.num_threads = config.model_config.num_threads;
+  c.model_config.provider = config.model_config.provider.c_str();
+  c.model_config.debug = config.model_config.debug;
+  c.model_config.model_type = config.model_config.model_type.c_str();
+  c.model_config.modeling_unit = config.model_config.modeling_unit.c_str();
+  c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str();
+  c.model_config.tokens_buf = config.model_config.tokens_buf.c_str();
+  c.model_config.tokens_buf_size = config.model_config.tokens_buf.size();
+
+  c.max_active_paths = config.max_active_paths;
+  c.num_trailing_blanks = config.num_trailing_blanks;
+  c.keywords_score = config.keywords_score;
+  c.keywords_threshold = config.keywords_threshold;
+  c.keywords_file = config.keywords_file.c_str();
+
+  auto p = SherpaOnnxCreateKeywordSpotter(&c);
+  return KeywordSpotter(p);
+}
+
+KeywordSpotter::KeywordSpotter(const SherpaOnnxKeywordSpotter *p)
+    : MoveOnly<KeywordSpotter, SherpaOnnxKeywordSpotter>(p) {}
+
+void KeywordSpotter::Destroy(const SherpaOnnxKeywordSpotter *p) const {
+  SherpaOnnxDestroyKeywordSpotter(p);
+}
+
+OnlineStream KeywordSpotter::CreateStream() const {
+  auto s = SherpaOnnxCreateKeywordStream(p_);
+  return OnlineStream{s};
+}
+
+OnlineStream KeywordSpotter::CreateStream(const std::string &keywords) const {
+  auto s = SherpaOnnxCreateKeywordStreamWithKeywords(p_, keywords.c_str());
+  return OnlineStream{s};
+}
+
+bool KeywordSpotter::IsReady(const OnlineStream *s) const {
+  return SherpaOnnxIsKeywordStreamReady(p_, s->Get());
+}
+
+void KeywordSpotter::Decode(const OnlineStream *s) const {
+  return SherpaOnnxDecodeKeywordStream(p_, s->Get());
+}
+
+void KeywordSpotter::Decode(const OnlineStream *ss, int32_t n) const {
+  if (n <= 0) {
+    return;
+  }
+
+  std::vector<const SherpaOnnxOnlineStream *> streams(n);
+  for (int32_t i = 0; i != n; ++n) {
+    streams[i] = ss[i].Get();
+  }
+
+  SherpaOnnxDecodeMultipleKeywordStreams(p_, streams.data(), n);
+}
+
+KeywordResult KeywordSpotter::GetResult(const OnlineStream *s) const {
+  auto r = SherpaOnnxGetKeywordResult(p_, s->Get());
+
+  KeywordResult ans;
+  ans.keyword = r->keyword;
+
+  ans.tokens.resize(r->count);
+  for (int32_t i = 0; i < r->count; ++i) {
+    ans.tokens[i] = r->tokens_arr[i];
+  }
+
+  if (r->timestamps) {
+    ans.timestamps.resize(r->count);
+    std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data());
+  }
+
+  ans.start_time = r->start_time;
+  ans.json = r->json;
+
+  SherpaOnnxDestroyKeywordResult(r);
+
+  return ans;
+}
+
+void KeywordSpotter::Reset(const OnlineStream *s) const {
+  SherpaOnnxResetKeywordStream(p_, s->Get());
+}
+
 }  // namespace sherpa_onnx::cxx

sherpa-onnx/c-api/cxx-api.h

@@ -406,6 +406,53 @@ class SHERPA_ONNX_API OfflineTts
   explicit OfflineTts(const SherpaOnnxOfflineTts *p);
 };
 
+// ============================================================
+// For Keyword Spotter
+// ============================================================
+
+struct KeywordResult {
+  std::string keyword;
+  std::vector<std::string> tokens;
+  std::vector<float> timestamps;
+  float start_time;
+  std::string json;
+};
+
+struct KeywordSpotterConfig {
+  FeatureConfig feat_config;
+  OnlineModelConfig model_config;
+  int32_t max_active_paths = 4;
+  int32_t num_trailing_blanks = 1;
+  float keywords_score = 1.0f;
+  float keywords_threshold = 0.25f;
+  std::string keywords_file;
+};
+
+class SHERPA_ONNX_API KeywordSpotter
+    : public MoveOnly<KeywordSpotter, SherpaOnnxKeywordSpotter> {
+ public:
+  static KeywordSpotter Create(const KeywordSpotterConfig &config);
+
+  void Destroy(const SherpaOnnxKeywordSpotter *p) const;
+
+  OnlineStream CreateStream() const;
+
+  OnlineStream CreateStream(const std::string &keywords) const;
+
+  bool IsReady(const OnlineStream *s) const;
+
+  void Decode(const OnlineStream *s) const;
+
+  void Decode(const OnlineStream *ss, int32_t n) const;
+
+  void Reset(const OnlineStream *s) const;
+
+  KeywordResult GetResult(const OnlineStream *s) const;
+
+ private:
+  explicit KeywordSpotter(const SherpaOnnxKeywordSpotter *p);
+};
+
 }  // namespace sherpa_onnx::cxx
 
 #endif  // SHERPA_ONNX_C_API_CXX_API_H_